diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-04 18:00:34 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-04 18:00:34 +0000 |
| commit | 3f619478f796eddbba6e39502fe941b285dd97b1 (patch) | |
| tree | e2c7b5777f728320e5b5542b6213fd3591ba51e2 /storage/myisam | |
| parent | Initial commit. (diff) | |
| download | mariadb-3f619478f796eddbba6e39502fe941b285dd97b1.tar.xz mariadb-3f619478f796eddbba6e39502fe941b285dd97b1.zip | |
Adding upstream version 1:10.11.6.upstream/1%10.11.6upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'storage/myisam')
112 files changed, 42802 insertions, 0 deletions
diff --git a/storage/myisam/CMakeLists.txt b/storage/myisam/CMakeLists.txt new file mode 100644 index 00000000..2f5d6211 --- /dev/null +++ b/storage/myisam/CMakeLists.txt @@ -0,0 +1,74 @@ +# Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA + +SET(MYISAM_SOURCES ft_boolean_search.c ft_nlq_search.c ft_parser.c ft_static.c + ha_myisam.cc + ft_myisam.c + ft_stopwords.c ft_update.c mi_cache.c mi_changed.c mi_check.c + mi_checksum.c mi_close.c mi_create.c mi_dbug.c mi_delete.c + mi_delete_all.c mi_delete_table.c mi_dynrec.c mi_extra.c mi_info.c + mi_key.c mi_keycache.c mi_locking.c mi_log.c mi_open.c + mi_packrec.c mi_page.c mi_panic.c mi_preload.c mi_range.c mi_rename.c + mi_rfirst.c mi_rlast.c mi_rnext.c mi_rnext_same.c mi_rprev.c mi_rrnd.c + mi_rsame.c mi_rsamepos.c mi_scan.c mi_search.c mi_static.c mi_statrec.c + mi_unique.c mi_update.c mi_write.c rt_index.c rt_key.c rt_mbr.c + rt_split.c sort.c sp_key.c mi_extrafunc.h myisamdef.h + rt_index.h mi_rkey.c) + +IF(CMAKE_SYSTEM_NAME MATCHES AIX) + # Workaround linker bug on AIX + SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-berok") +ENDIF() + +MYSQL_ADD_PLUGIN(myisam ${MYISAM_SOURCES} + STORAGE_ENGINE + MANDATORY + RECOMPILE_FOR_EMBEDDED) + +TARGET_LINK_LIBRARIES(myisam mysys) + +MYSQL_ADD_EXECUTABLE(myisam_ftdump myisam_ftdump.c COMPONENT Server) +TARGET_LINK_LIBRARIES(myisam_ftdump myisam) + +MYSQL_ADD_EXECUTABLE(myisamchk myisamchk.c COMPONENT Server) +TARGET_LINK_LIBRARIES(myisamchk myisam) + +MYSQL_ADD_EXECUTABLE(myisamlog myisamlog.c COMPONENT Server) +TARGET_LINK_LIBRARIES(myisamlog myisam) + +MYSQL_ADD_EXECUTABLE(myisampack myisampack.c COMPONENT Server) +TARGET_LINK_LIBRARIES(myisampack myisam) + +IF(WITH_UNIT_TESTS) + ADD_EXECUTABLE(mi_test1 mi_test1.c) + TARGET_LINK_LIBRARIES(mi_test1 myisam) + + ADD_EXECUTABLE(mi_test2 mi_test2.c) + TARGET_LINK_LIBRARIES(mi_test2 myisam) + + ADD_EXECUTABLE(mi_test3 mi_test3.c) + TARGET_LINK_LIBRARIES(mi_test3 myisam) + + ADD_EXECUTABLE(sp_test sp_test.c) + TARGET_LINK_LIBRARIES(sp_test myisam) + + ADD_EXECUTABLE(rt_test rt_test.c) + TARGET_LINK_LIBRARIES(rt_test myisam) +ENDIF() + +IF (MSVC) + SET_TARGET_PROPERTIES(myisamchk myisampack PROPERTIES LINK_FLAGS "setargv.obj") +ENDIF() + diff --git a/storage/myisam/ChangeLog b/storage/myisam/ChangeLog new file mode 100644 index 00000000..504202be --- /dev/null +++ b/storage/myisam/ChangeLog @@ -0,0 +1,150 @@ +2000-11-27 Michael Widenius <monty@mysql.com> + +* Changed mi_create.c to use less stack. + +2000-08-23 Michael Widenius <monty@mysql.com> + +* Fixed bug when comparing DECIMAL/NUMERIC key parts. + +2000-08-17 Michael Widenius <monty@mysql.com> + +* Add a new flag in share.staus so that we can quickly check if a table + is analyzed or not! + +2000-07-02 Michael Widenius <monty@mysql.com> + +* Added safety margin to guard against full index file. + +2000-05-22 Michael Widenius <monty@mysql.com> + +* Fixed that --join works with myisampack. + +2000-05-14 Michael Widenius <monty@mysql.com> + +* Don't lock datafile during myisamchk (only indexfile is locked; This is good + enough for all MyISAM functions); This made it possible to close datafile + in rep_by_sort(). + +2000-05-04 Michael Widenius <monty@mysql.com> + +* Fixed bug in code that scanned after rows in a crashed table. + This could cause an infinite loop when repairing tables. + +2000-04-26 Michael Widenius <monty@mysql.com> + +* Fixed bug when doing read_next after a delete/insert which balanced key + pages (In this case one internal buffer was wrongly reused) + +2000-04-21 Michael Widenius <monty@tik.pp.sci.fi> + +* Changed mi_find_halfpos() to return key, key_length and pos after key. +* Don't join or split key buffers in the middle when inserting a key + that is bigger than all other keys; This will improve inserts when + doing these in sorted order. + +2000-04-04 Michael Widenius <monty@mysql.com> + +* Added support for different languages on key part level. + +2000-02-23 Michael Widenius <monty@monty.pp.sci.fi> + +* Fixed that myisamchk works properly with RAID. + +2000-02-07 Michael Widenius <monty@tik.pp.sci.fi> + +* Added delete and rename of tables (works with RAID tables) + +2000-01-29 Michael Widenius <monty@monty.pp.sci.fi> + +* Fixed the sorting of index works with prefix-packed keys. + +1999-11-24 Michael Widenius <monty@monty.pp.sci.fi> + +* Fixed that DECIMAL() keys are sorted correct for negative numbers. + +1999-11-22 Michael Widenius <monty@monty.pp.sci.fi> + +* removed 'NO_LOCKING' macros. +* Added function mi_rnext_same +* Added support for concurrent reads. + +1999-11-05 Michael Widenius <monty@tik.pp.sci.fi> + +* Added function mi_scan(). +* Changed all functions to return error number in case of errors. + +1999-08-17 Michael Widenius <monty@tik.pp.sci.fi> + +* Added option DELAY_KEY_WRITE to tables and mi_open() + +1999-08-10 Michael Widenius <monty@tik.pp.sci.fi> + +* Added support of HA_READ_PREFIX_LAST to mi_rkey(). This finds the last + row with the given prefix. + +Mon Aug 2 13:54:35 1999 Michael Widenius <monty@bitch.pp.sci.fi> + +* Added data- and key-file-length to myisamchk. +* Fixed some problems with null and space packed keys. + +1999-07-15 Michael Widenius <monty@tik.pp.sci.fi> + +* The following options are for COUNT(DISTINCT ..) +* Added option HA_EXTRA_NO_ROWS; In this case only the index tree is updated +* Added mi_delete_all_rows() + +1999-07-13 Michael Widenius <monty@tik.pp.sci.fi> + +* Added special handling of tempoary tables + +1999-06-12 Michael Widenius <monty@monty.pp.sci.fi> + +* Added optional checksum for file and for each dynamic-length row +* Added unique constraint checking + +1999-05-06 Michael Widenius <monty@tik.pp.sci.fi> + +* All index blocks of the same size now share the same key block delete link + +1999-03-17 Michael Widenius <monty@monty.pp.sci.fi> + +* Different key packing code depending on if the first key part + is a variable length column (space packed, BLOB or VARCHAR) +* The create interface allows one to specify a key segment to start and + end one a specific bit. (The bit handling isn't yet implemented) +* Added more tests to 'test1' + +1999-03-16 Michael Widenius <monty@monty.pp.sci.fi> + +* Added option -m to myisamchk as an alternative to -e (-m is faster but + not as quite as safe as -e). +* Added option --fast to not check not changed tables. +* The first update will set a bit that the table has been changed. +* The first update to a table increments a 'open_count' field. This will + be reset on close. This will allow myisamchk to find tables that hasn't + been properly closed! +* Support for true VARCHAR columns + +1999-03-01 Michael Widenius <monty@monty.pp.sci.fi> + +* Dynamic length blocks are double linked to allow easy reallocation + of block lengths. This will help that the dynamic length data will not be + as fragmented as with ISAM. +* Extended mypack_isam to compress BLOB/TEXT columns. +* Allow keys on BLOB. + +1999-02-06 Michael Widenius <monty@monty.pp.sci.fi> + +* Keys, key pointers and all varibles in the index file are stored in + high-endian-order to get better compression. +* Allow NULL on keys + +1998-10-29 Michael Widenius <monty@monty.pp.sci.fi> + +* All data is stored in low-endian order + (This means that the files will be architecture and OS independent) +* All record numbers are now of type 'ha_rows' and file pointer are now of type + my_off_t. One can use files with 32-64 bit pointers with a 32bit or 64bit + record handling. + Currently the code is limited to 5 bytes pointers, but this is real easy + to change. diff --git a/storage/myisam/NEWS b/storage/myisam/NEWS new file mode 100644 index 00000000..942926a0 --- /dev/null +++ b/storage/myisam/NEWS @@ -0,0 +1,66 @@ +New features compared to NISAM: + +- All file positions have type my_off_t; This enables one to use big + files (2^63 byte) by defining my_off_t to be longlong on OS that supports + big files. +- When creating a table, one can now specify the maximum data file length. + This will be used to calculate the length of row pointers. +- All key segments have their own language definition. +- Some changes to support more types: + The biggest change is that the interface allows MY_ISAM will support + variable length integer types. (Only the interface is implemented) +- All data is stored with low byte first; This makes the data machine + independent. +- All number keys are stored with high byte first to give better packing. +- Support for a true VARCHAR type; A VARCHAR column starts with a length + stored on 2 bytes. +- Tables with VARCHAR may have fixed or dynamic record length. +- There are now 2 different ways to pack keys: + - If the first key part is a space stripped CHAR, a VARCHAR or a BLOB the + 'packed' method is used. This only prefix-compresses the first + key part. + - In other cases prefix packing is used (This also includes the record + pointer into the prefix packing). A key may in the best case be + packed on 2 bytes. +- VARCHAR and CHAR may be up to 65K +- Index on BLOB and VARCHAR. +- One can now have NULL in an index. This takes 0-1 bytes / key. +- MYISAM will allow one to specify one AUTO_INCREMENT column; MYISAM will + automatically update this on INSERT/UPDATE. The AUTO_INCREMENT value can be + reset with myisamchk. +- Max key length will be 500 by default; In cases of longer keys than 250, + a bigger key block size than the default of 1024 byes is used for this key. +- Max number of keys enlarged to 32 as default. This can be enlarged to 64 + without having to recompile myisamchk. +- There is a flag in the MYISAM header that tells if the index file (.MYI) + was closed correctly. +- myisamchk will now mark tables as checked. 'myisamchk --fast' will only + check those tables that doesn't have this mark. +- 'myisamchk -a' stores statistic for key parts (and not only for whole keys + as in NISAM). +- Dynamic size rows will now be much less fragmented when mixing deletes with + update and insert. This is done by automatically combining adjacent deleted + blocks and by extending blocks if the next block is deleted. +- For dynamic size rows, the delete link contains a pointer to itself + (to make repairs easier). +- myisampack (called pack_isam in NISAM) can pack BLOB and VARCHAR + columns. +- One can now disable any key from update; In NISAM one could only disable + the last x keys. +- One can have a UNIQUE constraint on anything (including BLOBS). + This is implemented by a key that contains a hashed number of the whole + record and before inserting a new record, MyISAM will check all records + with the same hash for dupplicates. +- When creating the table, one can define that MyISAM should maintain + a CRC for the whole table (to make isamchk even better). In the case of + dynamic size rows, one will in this case get 1 byte checksum for each row. + (This is a great help for debugging, but it can also be used to keep + MyISAM table 'extra' safe. +- Temporary tables will not write not flushed keys to file on close and + not wait on 'disk full' conditions. + +Interface changes compared to NISAM: + +- mi_create() + - keyinfo->seg must be allocated explicitly. + - One must put number of key segments in keyinfo diff --git a/storage/myisam/ft_boolean_search.c b/storage/myisam/ft_boolean_search.c new file mode 100644 index 00000000..f69f1869 --- /dev/null +++ b/storage/myisam/ft_boolean_search.c @@ -0,0 +1,1059 @@ +/* Copyright (c) 2001, 2016, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +/* TODO: add caching - pre-read several index entries at once */ + +/* + Added optimization for full-text queries with plus-words. It was + implemented by sharing maximal document id (max_docid) variable + inside plus subtree. max_docid could be used by any word in plus + subtree, but it could be updated by plus-word only. + + Fulltext "smarter index merge" optimization assumes that rows + it gets are ordered by doc_id. That is not the case when we + search for a word with truncation operator. It may return + rows in random order. Thus we may not use "smarter index merge" + optimization with "trunc-words". + + The idea is: there is no need to search for docid smaller than + biggest docid inside current plus subtree or any upper plus subtree. + + Examples: + +word1 word2 + share same max_docid + max_docid updated by word1 + +word1 +(word2 word3) + share same max_docid + max_docid updated by word1 + +(word1 -word2) +(+word3 word4) + share same max_docid + max_docid updated by word3 + +word1 word2 (+word3 word4 (+word5 word6)) + three subexpressions (including the top-level one), + every one has its own max_docid, updated by its plus word. + but for the search word6 uses + max(word1.max_docid, word3.max_docid, word5.max_docid), + while word4 uses, accordingly, + max(word1.max_docid, word3.max_docid). +*/ + +#define FT_CORE +#include "ftdefs.h" + +/* search with boolean queries */ + +static double _wghts[11]= +{ + 0.131687242798354, + 0.197530864197531, + 0.296296296296296, + 0.444444444444444, + 0.666666666666667, + 1.000000000000000, + 1.500000000000000, + 2.250000000000000, + 3.375000000000000, + 5.062500000000000, + 7.593750000000000}; +static double *wghts=_wghts+5; /* wghts[i] = 1.5**i */ + +static double _nwghts[11]= +{ + -0.065843621399177, + -0.098765432098766, + -0.148148148148148, + -0.222222222222222, + -0.333333333333334, + -0.500000000000000, + -0.750000000000000, + -1.125000000000000, + -1.687500000000000, + -2.531250000000000, + -3.796875000000000}; +static double *nwghts=_nwghts+5; /* nwghts[i] = -0.5*1.5**i */ + +#define FTB_FLAG_TRUNC 1 +/* At most one of the following flags can be set */ +#define FTB_FLAG_YES 2 +#define FTB_FLAG_NO 4 +#define FTB_FLAG_WONLY 8 + +typedef struct st_ftb_expr FTB_EXPR; +struct st_ftb_expr +{ + FTB_EXPR *up; + uint flags; +/* ^^^^^^^^^^^^^^^^^^ FTB_{EXPR,WORD} common section */ + my_off_t docid[2]; + my_off_t max_docid; + float weight; + float cur_weight; + LIST *phrase; /* phrase words */ + LIST *document; /* for phrase search */ + uint yesses; /* number of "yes" words matched */ + uint nos; /* number of "no" words matched */ + uint ythresh; /* number of "yes" words in expr */ + uint yweaks; /* number of "yes" words for scan only */ +}; + +typedef struct st_ftb_word +{ + FTB_EXPR *up; + uint flags; +/* ^^^^^^^^^^^^^^^^^^ FTB_{EXPR,WORD} common section */ + my_off_t docid[2]; /* for index search and for scan */ + my_off_t key_root; + FTB_EXPR *max_docid_expr; + MI_KEYDEF *keyinfo; + struct st_ftb_word *prev; + float weight; + uint ndepth; + uint len; + uchar off; + uchar word[1]; +} FTB_WORD; + +typedef struct st_ft_info +{ + struct _ft_vft *please; + MI_INFO *info; + CHARSET_INFO *charset; + FTB_EXPR *root; + FTB_WORD **list; + FTB_WORD *last_word; + MEM_ROOT mem_root; + QUEUE queue; + TREE no_dupes; + my_off_t lastpos; + uint keynr; + uchar with_scan; + enum { UNINITIALIZED, READY, INDEX_SEARCH, INDEX_DONE } state; +} FTB; + +static int FTB_WORD_cmp(my_off_t *v, FTB_WORD *a, FTB_WORD *b) +{ + int i; + + /* if a==curdoc, take it as a < b */ + if (v && a->docid[0] == *v) + return -1; + + /* ORDER BY docid, ndepth DESC */ + i=CMP_NUM(a->docid[0], b->docid[0]); + if (!i) + i=CMP_NUM(b->ndepth,a->ndepth); + return i; +} + +static int FTB_WORD_cmp_list(CHARSET_INFO *cs, FTB_WORD **a, FTB_WORD **b) +{ + /* ORDER BY word, ndepth */ + int i= ha_compare_word(cs, (uchar*) (*a)->word + 1, (*a)->len - 1, + (uchar*) (*b)->word + 1, (*b)->len - 1); + if (!i) + i= CMP_NUM((*a)->ndepth, (*b)->ndepth); + return i; +} + + +typedef struct st_my_ftb_param +{ + FTB *ftb; + FTB_EXPR *ftbe; + uchar *up_quot; + uint depth; +} MY_FTB_PARAM; + + +static int ftb_query_add_word(MYSQL_FTPARSER_PARAM *param, + const char *word, int word_len, + MYSQL_FTPARSER_BOOLEAN_INFO *info) +{ + MY_FTB_PARAM *ftb_param= param->mysql_ftparam; + FTB_WORD *ftbw; + FTB_EXPR *ftbe, *tmp_expr; + FT_WORD *phrase_word; + LIST *tmp_element; + int r= info->weight_adjust; + float weight= (float) + (info->wasign ? nwghts : wghts)[(r>5)?5:((r<-5)?-5:r)]; + + switch (info->type) { + case FT_TOKEN_WORD: + ftbw= (FTB_WORD *)alloc_root(&ftb_param->ftb->mem_root, + sizeof(FTB_WORD) + HA_MAX_KEY_BUFF); + ftbw->len= word_len + 1; + ftbw->flags= 0; + ftbw->off= 0; + if (info->yesno > 0) ftbw->flags|= FTB_FLAG_YES; + if (info->yesno < 0) ftbw->flags|= FTB_FLAG_NO; + if (info->trunc) ftbw->flags|= FTB_FLAG_TRUNC; + ftbw->weight= weight; + ftbw->up= ftb_param->ftbe; + ftbw->docid[0]= ftbw->docid[1]= HA_OFFSET_ERROR; + ftbw->ndepth= (info->yesno < 0) + ftb_param->depth; + ftbw->key_root= HA_OFFSET_ERROR; + memcpy(ftbw->word + 1, word, word_len); + ftbw->word[0]= word_len; + if (info->yesno > 0) ftbw->up->ythresh++; + ftb_param->ftb->queue.max_elements++; + ftbw->prev= ftb_param->ftb->last_word; + ftb_param->ftb->last_word= ftbw; + ftb_param->ftb->with_scan|= (info->trunc & FTB_FLAG_TRUNC); + for (tmp_expr= ftb_param->ftbe; tmp_expr->up; tmp_expr= tmp_expr->up) + if (! (tmp_expr->flags & FTB_FLAG_YES)) + break; + ftbw->max_docid_expr= tmp_expr; + /* fall through */ + case FT_TOKEN_STOPWORD: + if (! ftb_param->up_quot) break; + phrase_word= (FT_WORD *)alloc_root(&ftb_param->ftb->mem_root, sizeof(FT_WORD)); + tmp_element= (LIST *)alloc_root(&ftb_param->ftb->mem_root, sizeof(LIST)); + phrase_word->pos= (uchar*) word; + phrase_word->len= word_len; + tmp_element->data= (void *)phrase_word; + ftb_param->ftbe->phrase= list_add(ftb_param->ftbe->phrase, tmp_element); + /* Allocate document list at this point. + It allows to avoid huge amount of allocs/frees for each row.*/ + tmp_element= (LIST *)alloc_root(&ftb_param->ftb->mem_root, sizeof(LIST)); + tmp_element->data= alloc_root(&ftb_param->ftb->mem_root, sizeof(FT_WORD)); + ftb_param->ftbe->document= + list_add(ftb_param->ftbe->document, tmp_element); + break; + case FT_TOKEN_LEFT_PAREN: + ftbe=(FTB_EXPR *)alloc_root(&ftb_param->ftb->mem_root, sizeof(FTB_EXPR)); + ftbe->flags= 0; + if (info->yesno > 0) ftbe->flags|= FTB_FLAG_YES; + if (info->yesno < 0) ftbe->flags|= FTB_FLAG_NO; + ftbe->weight= weight; + ftbe->up= ftb_param->ftbe; + ftbe->max_docid= ftbe->ythresh= ftbe->yweaks= 0; + ftbe->docid[0]= ftbe->docid[1]= HA_OFFSET_ERROR; + ftbe->phrase= NULL; + ftbe->document= 0; + if (info->quot) ftb_param->ftb->with_scan|= 2; + if (info->yesno > 0) ftbe->up->ythresh++; + ftb_param->ftbe= ftbe; + ftb_param->depth++; + ftb_param->up_quot= (uchar*) info->quot; + break; + case FT_TOKEN_RIGHT_PAREN: + if (ftb_param->ftbe->document) + { + /* Circuit document list */ + for (tmp_element= ftb_param->ftbe->document; + tmp_element->next; tmp_element= tmp_element->next) /* no-op */; + tmp_element->next= ftb_param->ftbe->document; + ftb_param->ftbe->document->prev= tmp_element; + } + info->quot= 0; + if (ftb_param->ftbe->up) + { + DBUG_ASSERT(ftb_param->depth); + ftb_param->ftbe= ftb_param->ftbe->up; + ftb_param->depth--; + ftb_param->up_quot= 0; + } + break; + case FT_TOKEN_EOF: + default: + break; + } + return(0); +} + + +static int ftb_parse_query_internal(MYSQL_FTPARSER_PARAM *param, + const char *query, int len) +{ + MY_FTB_PARAM *ftb_param= param->mysql_ftparam; + MYSQL_FTPARSER_BOOLEAN_INFO info; + CHARSET_INFO *cs= ftb_param->ftb->charset; + const uchar **start= (const uchar**) &query; + uchar *end= (uchar*) query + len; + FT_WORD w; + + info.prev= ' '; + info.quot= 0; + while (ft_get_word(cs, start, end, &w, &info)) + param->mysql_add_word(param, (char*) w.pos, (int)w.len, &info); + return(0); +} + + +static int _ftb_parse_query(FTB *ftb, uchar *query, uint len, + struct st_mysql_ftparser *parser) +{ + MYSQL_FTPARSER_PARAM *param; + MY_FTB_PARAM ftb_param; + DBUG_ENTER("_ftb_parse_query"); + DBUG_ASSERT(parser); + + if (ftb->state != UNINITIALIZED) + DBUG_RETURN(0); + if (! (param= ftparser_call_initializer(ftb->info, ftb->keynr, 0))) + DBUG_RETURN(1); + + ftb_param.ftb= ftb; + ftb_param.depth= 0; + ftb_param.ftbe= ftb->root; + ftb_param.up_quot= 0; + + param->mysql_parse= ftb_parse_query_internal; + param->mysql_add_word= ftb_query_add_word; + param->mysql_ftparam= (void *)&ftb_param; + param->cs= ftb->charset; + param->doc= (char*) query; + param->length= len; + param->flags= 0; + param->mode= MYSQL_FTPARSER_FULL_BOOLEAN_INFO; + DBUG_RETURN(parser->parse(param)); +} + + +static int _ftb_no_dupes_cmp(void* not_used __attribute__((unused)), + const void *a,const void *b) +{ + return CMP_NUM((*((my_off_t*)a)), (*((my_off_t*)b))); +} + +/* + When performing prefix search (a word with truncation operator), we + must preserve original prefix to ensure that characters which may be + expanded/contracted do not break the prefix. This is done by storing + newly found key immediately after the original word in ftbw->word + buffer. + + ftbw->word= LENGTH WORD [ LENGTH1 WORD1 ] WEIGHT REFERENCE + LENGTH - 1 byte, length of the WORD + WORD - LENGTH bytes, the word itself + LENGTH1 - 1 byte, length of the WORD1, present in case of prefix search + WORD1 - LENGTH bytes, the word itself, present in case of prefix search + WEIGHT - 4 bytes (HA_FT_WLEN), either weight or number of subkeys + REFERENCE - rec_reflength bytes, pointer to the record + + returns 1 if the search was finished (must-word wasn't found) +*/ +static int _ft2_search_no_lock(FTB *ftb, FTB_WORD *ftbw, my_bool init_search) +{ + int r; + int subkeys=1; + my_bool can_go_down; + MI_INFO *info=ftb->info; + uint UNINIT_VAR(off), extra= HA_FT_WLEN + info->s->rec_reflength; + uchar *lastkey_buf=ftbw->word+ftbw->off; + + if (ftbw->flags & FTB_FLAG_TRUNC) + lastkey_buf+=ftbw->len; + + if (init_search) + { + ftbw->key_root=info->s->state.key_root[ftb->keynr]; + ftbw->keyinfo=info->s->keyinfo+ftb->keynr; + + r=_mi_search(info, ftbw->keyinfo, (uchar*) ftbw->word, ftbw->len, + SEARCH_FIND | SEARCH_BIGGER, ftbw->key_root); + } + else + { + uint sflag= SEARCH_BIGGER; + my_off_t max_docid=0; + FTB_EXPR *tmp; + + for (tmp= ftbw->max_docid_expr; tmp; tmp= tmp->up) + set_if_bigger(max_docid, tmp->max_docid); + + if (ftbw->docid[0] < max_docid) + { + sflag|= SEARCH_SAME; + _mi_dpointer(info, (uchar*) (lastkey_buf + HA_FT_WLEN + + (ftbw->off ? 0 : lastkey_buf[0] + 1)), + max_docid); + } + r=_mi_search(info, ftbw->keyinfo, (uchar*) lastkey_buf, + USE_WHOLE_KEY, sflag, ftbw->key_root); + } + + can_go_down=(!ftbw->off && (init_search || (ftbw->flags & FTB_FLAG_TRUNC))); + /* Skip rows inserted by concurrent insert */ + while (!r) + { + if (can_go_down) + { + /* going down ? */ + off=info->lastkey_length-extra; + subkeys=ft_sintXkorr(info->lastkey+off); + } + if (subkeys<0 || info->lastpos < info->state->data_file_length) + break; + r= _mi_search_next(info, ftbw->keyinfo, info->lastkey, + info->lastkey_length, + SEARCH_BIGGER, ftbw->key_root); + } + + if (!r && !ftbw->off) + { + r= ha_compare_word_or_prefix(ftb->charset, + info->lastkey + 1, + info->lastkey_length - extra - 1, + (uchar*) ftbw->word + 1, + ftbw->len - 1, + (my_bool) (ftbw->flags & FTB_FLAG_TRUNC)); + } + + if (r) /* not found */ + { + if (!ftbw->off || !(ftbw->flags & FTB_FLAG_TRUNC)) + { + ftbw->docid[0]=HA_OFFSET_ERROR; + if ((ftbw->flags & FTB_FLAG_YES) && ftbw->up->up==0) + { + /* + This word MUST BE present in every document returned, + so we can stop the search right now + */ + ftb->state=INDEX_DONE; + return 1; /* search is done */ + } + else + return 0; + } + + /* + Going up to the first-level tree to continue search there. + Only done when performing prefix search. + + Key buffer data pointer as well as docid[0] may be smaller + than values we got while searching first-level tree. Thus + they must be restored to original values to avoid dead-loop, + when subsequent search for a bigger value eventually ends up + in this same second-level tree. + */ + _mi_dpointer(info, (uchar*) (lastkey_buf+HA_FT_WLEN), ftbw->key_root); + ftbw->docid[0]= ftbw->key_root; + ftbw->key_root=info->s->state.key_root[ftb->keynr]; + ftbw->keyinfo=info->s->keyinfo+ftb->keynr; + ftbw->off=0; + return _ft2_search_no_lock(ftb, ftbw, 0); + } + + /* matching key found */ + memcpy(lastkey_buf, info->lastkey, info->lastkey_length); + if (lastkey_buf == ftbw->word) + ftbw->len=info->lastkey_length-extra; + + /* going down ? */ + if (subkeys<0) + { + /* + yep, going down, to the second-level tree + TODO here: subkey-based optimization + */ + ftbw->off=off; + ftbw->key_root=info->lastpos; + ftbw->keyinfo=& info->s->ft2_keyinfo; + r=_mi_search_first(info, ftbw->keyinfo, ftbw->key_root); + DBUG_ASSERT(r==0); /* found something */ + memcpy(lastkey_buf+off, info->lastkey, info->lastkey_length); + } + ftbw->docid[0]=info->lastpos; + if (ftbw->flags & FTB_FLAG_YES && !(ftbw->flags & FTB_FLAG_TRUNC)) + ftbw->max_docid_expr->max_docid= info->lastpos; + return 0; +} + +static int _ft2_search(FTB *ftb, FTB_WORD *ftbw, my_bool init_search) +{ + int r; + MYISAM_SHARE *share= ftb->info->s; + if (share->concurrent_insert) + mysql_rwlock_rdlock(&share->key_root_lock[ftb->keynr]); + r= _ft2_search_no_lock(ftb, ftbw, init_search); + if (share->concurrent_insert) + mysql_rwlock_unlock(&share->key_root_lock[ftb->keynr]); + return r; +} + +static void _ftb_init_index_search(FT_INFO *ftb) +{ + int i; + FTB_WORD *ftbw; + + if (ftb->state == UNINITIALIZED || ftb->keynr == NO_SUCH_KEY) + return; + ftb->state=INDEX_SEARCH; + + for (i= queue_last_element(&ftb->queue); + i >= (int) queue_first_element(&ftb->queue); + i--) + { + ftbw=(FTB_WORD *)(queue_element(&ftb->queue, i)); + + if (ftbw->flags & FTB_FLAG_TRUNC) + { + /* + special treatment for truncation operator + 1. there are some (besides this) +words + | no need to search in the index, it can never ADD new rows + | to the result, and to remove half-matched rows we do scan anyway + 2. -trunc* + | same as 1. + 3. in 1 and 2, +/- need not be on the same expr. level, + but can be on any upper level, as in +word +(trunc1* trunc2*) + 4. otherwise + | We have to index-search for this prefix. + | It may cause duplicates, as in the index (sorted by <word,docid>) + | <aaaa,row1> + | <aabb,row2> + | <aacc,row1> + | Searching for "aa*" will find row1 twice... + */ + FTB_EXPR *ftbe; + for (ftbe=(FTB_EXPR*)ftbw; + ftbe->up && !(ftbe->up->flags & FTB_FLAG_TRUNC); + ftbe->up->flags|= FTB_FLAG_TRUNC, ftbe=ftbe->up) + { + if (ftbe->flags & FTB_FLAG_NO || /* 2 */ + ftbe->up->ythresh - ftbe->up->yweaks > + (uint) MY_TEST(ftbe->flags & FTB_FLAG_YES)) /* 1 */ + { + FTB_EXPR *top_ftbe=ftbe->up; + ftbw->docid[0]=HA_OFFSET_ERROR; + for (ftbe=(FTB_EXPR *)ftbw; + ftbe != top_ftbe && !(ftbe->flags & FTB_FLAG_NO); + ftbe=ftbe->up) + ftbe->up->yweaks++; + ftbe=0; + break; + } + } + if (!ftbe) + continue; + /* 4 */ + if (!is_tree_inited(& ftb->no_dupes)) + init_tree(& ftb->no_dupes,0,0,sizeof(my_off_t), + _ftb_no_dupes_cmp,0,0,MYF(0)); + else + reset_tree(& ftb->no_dupes); + } + + ftbw->off=0; /* in case of reinit */ + if (_ft2_search(ftb, ftbw, 1)) + return; + } + queue_fix(& ftb->queue); +} + + +FT_INFO * ft_init_boolean_search(MI_INFO *info, uint keynr, uchar *query, + uint query_len, CHARSET_INFO *cs) +{ + FTB *ftb; + FTB_EXPR *ftbe; + FTB_WORD *ftbw; + + if (!(ftb=(FTB *)my_malloc(mi_key_memory_FTB, sizeof(FTB), MYF(MY_WME)))) + return 0; + ftb->please= (struct _ft_vft *) & _ft_vft_boolean; + ftb->state=UNINITIALIZED; + ftb->info=info; + ftb->keynr=keynr; + ftb->charset=cs; + DBUG_ASSERT(keynr==NO_SUCH_KEY || cs == info->s->keyinfo[keynr].seg->charset); + ftb->with_scan=0; + ftb->lastpos=HA_OFFSET_ERROR; + bzero(& ftb->no_dupes, sizeof(TREE)); + ftb->last_word= 0; + + init_alloc_root(mi_key_memory_FTB, &ftb->mem_root, 1024, 1024, MYF(0)); + ftb->queue.max_elements= 0; + if (!(ftbe=(FTB_EXPR *)alloc_root(&ftb->mem_root, sizeof(FTB_EXPR)))) + goto err; + ftbe->weight=1; + ftbe->flags=FTB_FLAG_YES; + ftbe->nos=1; + ftbe->up=0; + ftbe->max_docid= ftbe->ythresh= ftbe->yweaks= 0; + ftbe->docid[0]=ftbe->docid[1]=HA_OFFSET_ERROR; + ftbe->phrase= NULL; + ftbe->document= 0; + ftb->root=ftbe; + if (unlikely(_ftb_parse_query(ftb, query, query_len, + keynr == NO_SUCH_KEY ? &ft_default_parser : + info->s->keyinfo[keynr].parser))) + goto err; + /* + Hack: instead of init_queue, we'll use reinit queue to be able + to alloc queue with alloc_root() + */ + if (! (ftb->queue.root= (uchar **)alloc_root(&ftb->mem_root, + (ftb->queue.max_elements + 1) * + sizeof(void *)))) + goto err; + reinit_queue(&ftb->queue, ftb->queue.max_elements, 0, 0, + (int (*)(void*, uchar*, uchar*))FTB_WORD_cmp, 0, 0, 0); + for (ftbw= ftb->last_word; ftbw; ftbw= ftbw->prev) + queue_insert(&ftb->queue, (uchar *)ftbw); + ftb->list=(FTB_WORD **)alloc_root(&ftb->mem_root, + sizeof(FTB_WORD *)*ftb->queue.elements); + memcpy(ftb->list, &queue_top(&ftb->queue), sizeof(FTB_WORD *)*ftb->queue.elements); + my_qsort2(ftb->list, ftb->queue.elements, sizeof(FTB_WORD *), + (qsort2_cmp)FTB_WORD_cmp_list, (void*)ftb->charset); + if (ftb->queue.elements<2) ftb->with_scan &= ~FTB_FLAG_TRUNC; + ftb->state=READY; + return ftb; +err: + free_root(& ftb->mem_root, MYF(0)); + my_free(ftb); + return 0; +} + + +typedef struct st_my_ftb_phrase_param +{ + LIST *phrase; + LIST *document; + CHARSET_INFO *cs; + uint phrase_length; + uint document_length; + uint match; +} MY_FTB_PHRASE_PARAM; + + +static int ftb_phrase_add_word(MYSQL_FTPARSER_PARAM *param, + const char *word, int word_len, + MYSQL_FTPARSER_BOOLEAN_INFO *boolean_info __attribute__((unused))) +{ + MY_FTB_PHRASE_PARAM *phrase_param= param->mysql_ftparam; + FT_WORD *w= (FT_WORD *)phrase_param->document->data; + LIST *phrase, *document; + w->pos= (uchar*) word; + w->len= word_len; + phrase_param->document= phrase_param->document->prev; + if (phrase_param->phrase_length > phrase_param->document_length) + { + phrase_param->document_length++; + return 0; + } + /* TODO: rewrite phrase search to avoid + comparing the same word twice. */ + for (phrase= phrase_param->phrase, document= phrase_param->document->next; + phrase; phrase= phrase->next, document= document->next) + { + FT_WORD *phrase_word= (FT_WORD *)phrase->data; + FT_WORD *document_word= (FT_WORD *)document->data; + if (my_strnncoll(phrase_param->cs, (uchar*) phrase_word->pos, + phrase_word->len, + (uchar*) document_word->pos, document_word->len)) + return 0; + } + phrase_param->match++; + return 0; +} + + +static int ftb_check_phrase_internal(MYSQL_FTPARSER_PARAM *param, + const char *document, int len) +{ + FT_WORD word; + MY_FTB_PHRASE_PARAM *phrase_param= param->mysql_ftparam; + const uchar *docend= (uchar*) document + len; + while (ft_simple_get_word(phrase_param->cs, (uchar**) &document, docend, + &word, FALSE)) + { + param->mysql_add_word(param, (char*) word.pos, (int)word.len, 0); + if (phrase_param->match) + break; + } + return 0; +} + + +/* + Checks if given buffer matches phrase list. + + SYNOPSIS + _ftb_check_phrase() + s0 start of buffer + e0 end of buffer + phrase broken into list phrase + cs charset info + + RETURN VALUE + 1 is returned if phrase found, 0 else. + -1 is returned if error occurs. +*/ + +static int _ftb_check_phrase(FTB *ftb, const uchar *document, uint len, + FTB_EXPR *ftbe, struct st_mysql_ftparser *parser) +{ + MY_FTB_PHRASE_PARAM ftb_param; + MYSQL_FTPARSER_PARAM *param; + DBUG_ENTER("_ftb_check_phrase"); + DBUG_ASSERT(parser); + + if (! (param= ftparser_call_initializer(ftb->info, ftb->keynr, 1))) + DBUG_RETURN(0); + + ftb_param.phrase= ftbe->phrase; + ftb_param.document= ftbe->document; + ftb_param.cs= ftb->charset; + ftb_param.phrase_length= list_length(ftbe->phrase); + ftb_param.document_length= 1; + ftb_param.match= 0; + + param->mysql_parse= ftb_check_phrase_internal; + param->mysql_add_word= ftb_phrase_add_word; + param->mysql_ftparam= (void *)&ftb_param; + param->cs= ftb->charset; + param->doc= (char *) document; + param->length= len; + param->flags= 0; + param->mode= MYSQL_FTPARSER_WITH_STOPWORDS; + if (unlikely(parser->parse(param))) + return -1; + DBUG_RETURN(ftb_param.match ? 1 : 0); +} + + +static int _ftb_climb_the_tree(FTB *ftb, FTB_WORD *ftbw, FT_SEG_ITERATOR *ftsi_orig) +{ + FT_SEG_ITERATOR ftsi; + FTB_EXPR *ftbe; + float weight=ftbw->weight; + int yn_flag= ftbw->flags, ythresh, mode=(ftsi_orig != 0); + my_off_t curdoc=ftbw->docid[mode]; + struct st_mysql_ftparser *parser= ftb->keynr == NO_SUCH_KEY ? + &ft_default_parser : + ftb->info->s->keyinfo[ftb->keynr].parser; + + for (ftbe=ftbw->up; ftbe; ftbe=ftbe->up) + { + ythresh = ftbe->ythresh - (mode ? 0 : ftbe->yweaks); + if (ftbe->docid[mode] != curdoc) + { + ftbe->cur_weight=0; + ftbe->yesses=ftbe->nos=0; + ftbe->docid[mode]=curdoc; + } + if (ftbe->nos) + break; + if (yn_flag & FTB_FLAG_YES) + { + weight /= ftbe->ythresh; + ftbe->cur_weight += weight; + if ((int) ++ftbe->yesses == ythresh) + { + yn_flag=ftbe->flags; + weight=ftbe->cur_weight*ftbe->weight; + if (mode && ftbe->phrase) + { + int found= 0; + + memcpy(&ftsi, ftsi_orig, sizeof(ftsi)); + while (_mi_ft_segiterator(&ftsi) && !found) + { + if (!ftsi.pos) + continue; + found= _ftb_check_phrase(ftb, ftsi.pos, ftsi.len, ftbe, parser); + if (unlikely(found < 0)) + return 1; + } + if (!found) + break; + } /* ftbe->quot */ + } + else + break; + } + else + if (yn_flag & FTB_FLAG_NO) + { + /* + NOTE: special sort function of queue assures that all + (yn_flag & FTB_FLAG_NO) != 0 + events for every particular subexpression will + "auto-magically" happen BEFORE all the + (yn_flag & FTB_FLAG_YES) != 0 events. So no + already matched expression can become not-matched again. + */ + ++ftbe->nos; + break; + } + else + { + if (ftbe->ythresh) + weight/=3; + ftbe->cur_weight += weight; + if ((int) ftbe->yesses < ythresh) + break; + if (!(yn_flag & FTB_FLAG_WONLY)) + yn_flag= ((int) ftbe->yesses++ == ythresh) ? ftbe->flags : FTB_FLAG_WONLY ; + weight*= ftbe->weight; + } + } + return 0; +} + + +int ft_boolean_read_next(FT_INFO *ftb, char *record) +{ + FTB_EXPR *ftbe; + FTB_WORD *ftbw; + MI_INFO *info=ftb->info; + my_off_t curdoc; + + if (ftb->state != INDEX_SEARCH && ftb->state != INDEX_DONE) + return -1; + + /* black magic ON */ + if ((int) _mi_check_index(info, ftb->keynr) < 0) + return my_errno; + if (_mi_readinfo(info, F_RDLCK, 1)) + return my_errno; + /* black magic OFF */ + + if (!ftb->queue.elements) + return my_errno=HA_ERR_END_OF_FILE; + + /* Attention!!! Address of a local variable is used here! See err: label */ + ftb->queue.first_cmp_arg=(void *)&curdoc; + + while (ftb->state == INDEX_SEARCH && + (curdoc=((FTB_WORD *)queue_top(& ftb->queue))->docid[0]) != + HA_OFFSET_ERROR) + { + while (curdoc == (ftbw=(FTB_WORD *)queue_top(& ftb->queue))->docid[0]) + { + if (unlikely(_ftb_climb_the_tree(ftb, ftbw, 0))) + { + my_errno= HA_ERR_OUT_OF_MEM; + goto err; + } + + /* update queue */ + _ft2_search(ftb, ftbw, 0); + queue_replace_top(&ftb->queue); + } + + ftbe=ftb->root; + if (ftbe->docid[0]==curdoc && ftbe->cur_weight>0 && + ftbe->yesses>=(ftbe->ythresh-ftbe->yweaks) && !ftbe->nos) + { + /* curdoc matched ! */ + if (is_tree_inited(&ftb->no_dupes) && + tree_insert(&ftb->no_dupes, &curdoc, 0, + ftb->no_dupes.custom_arg)->count >1) + /* but it managed already to get past this line once */ + continue; + + info->lastpos=curdoc; + /* Clear all states, except that the table was updated */ + info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + + if (!(*info->read_record)(info,curdoc, (uchar*) record)) + { + info->update|= HA_STATE_AKTIV; /* Record is read */ + if (ftb->with_scan && + ft_boolean_find_relevance(ftb,(uchar*) record,0)==0) + continue; /* no match */ + my_errno=0; + goto err; + } + goto err; + } + } + ftb->state=INDEX_DONE; + my_errno=HA_ERR_END_OF_FILE; +err: + ftb->queue.first_cmp_arg=(void *)0; + return my_errno; +} + + +typedef struct st_my_ftb_find_param +{ + FT_INFO *ftb; + FT_SEG_ITERATOR *ftsi; +} MY_FTB_FIND_PARAM; + + +static int ftb_find_relevance_add_word(MYSQL_FTPARSER_PARAM *param, + const char *word, int len, + MYSQL_FTPARSER_BOOLEAN_INFO *boolean_info __attribute__((unused))) +{ + MY_FTB_FIND_PARAM *ftb_param= param->mysql_ftparam; + FT_INFO *ftb= ftb_param->ftb; + FTB_WORD *ftbw; + int a, b, c; + /* + Find right-most element in the array of query words matching this + word from a document. + */ + for (a= 0, b= ftb->queue.elements, c= (a+b)/2; b-a>1; c= (a+b)/2) + { + ftbw= ftb->list[c]; + if (ha_compare_word_or_prefix(ftb->charset, (uchar*)word, len, + (uchar*)ftbw->word + 1, ftbw->len - 1, + (my_bool) (ftbw->flags & FTB_FLAG_TRUNC)) < 0) + b= c; + else + a= c; + } + /* + If there were no words with truncation operator, we iterate to the + beginning of an array until array element is equal to the word from + a document. This is done mainly because the same word may be + mentioned twice (or more) in the query. + + In case query has words with truncation operator we must iterate + to the beginning of the array. There may be non-matching query words + between matching word with truncation operator and the right-most + matching element. E.g., if we're looking for 'aaa15' in an array of + 'aaa1* aaa14 aaa15 aaa16'. + + Worse of that there still may be match even if the binary search + above didn't find matching element. E.g., if we're looking for + 'aaa15' in an array of 'aaa1* aaa14 aaa16'. The binary search will + stop at 'aaa16'. + */ + for (; c >= 0; c--) + { + ftbw= ftb->list[c]; + if (ha_compare_word_or_prefix(ftb->charset, (uchar*) word, len, + (uchar*) ftbw->word + 1, ftbw->len - 1, + (my_bool) (ftbw->flags & FTB_FLAG_TRUNC))) + { + if (ftb->with_scan & FTB_FLAG_TRUNC) + continue; + else + break; + } + if (ftbw->docid[1] == ftb->info->lastpos) + continue; + ftbw->docid[1]= ftb->info->lastpos; + if (unlikely(_ftb_climb_the_tree(ftb, ftbw, ftb_param->ftsi))) + return 1; + } + return(0); +} + + +static int ftb_find_relevance_parse(MYSQL_FTPARSER_PARAM *param, + const char *doc, int len) +{ + MY_FTB_FIND_PARAM *ftb_param= param->mysql_ftparam; + FT_INFO *ftb= ftb_param->ftb; + uchar *end= (uchar*) doc + len; + FT_WORD w; + while (ft_simple_get_word(ftb->charset, (uchar**) &doc, end, &w, TRUE)) + param->mysql_add_word(param, (char*) w.pos, (int)w.len, 0); + return(0); +} + + +float ft_boolean_find_relevance(FT_INFO *ftb, uchar *record, uint length) +{ + FTB_EXPR *ftbe; + FT_SEG_ITERATOR ftsi, ftsi2; + my_off_t docid=ftb->info->lastpos; + MY_FTB_FIND_PARAM ftb_param; + MYSQL_FTPARSER_PARAM *param; + struct st_mysql_ftparser *parser= ftb->keynr == NO_SUCH_KEY ? + &ft_default_parser : + ftb->info->s->keyinfo[ftb->keynr].parser; + + if (docid == HA_OFFSET_ERROR) + return -2.0; + if (!ftb->queue.elements) + return 0; + if (! (param= ftparser_call_initializer(ftb->info, ftb->keynr, 0))) + return 0; + + if (ftb->state != INDEX_SEARCH && docid <= ftb->lastpos) + { + FTB_EXPR *x; + uint i; + + for (i=0; i < ftb->queue.elements; i++) + { + ftb->list[i]->docid[1]=HA_OFFSET_ERROR; + for (x=ftb->list[i]->up; x; x=x->up) + x->docid[1]=HA_OFFSET_ERROR; + } + } + + ftb->lastpos=docid; + + if (ftb->keynr==NO_SUCH_KEY) + _mi_ft_segiterator_dummy_init(record, length, &ftsi); + else + _mi_ft_segiterator_init(ftb->info, ftb->keynr, record, &ftsi); + memcpy(&ftsi2, &ftsi, sizeof(ftsi)); + + ftb_param.ftb= ftb; + ftb_param.ftsi= &ftsi2; + param->mysql_parse= ftb_find_relevance_parse; + param->mysql_add_word= ftb_find_relevance_add_word; + param->mysql_ftparam= (void *)&ftb_param; + param->flags= 0; + param->cs= ftb->charset; + param->mode= MYSQL_FTPARSER_SIMPLE_MODE; + while (_mi_ft_segiterator(&ftsi)) + { + if (!ftsi.pos) + continue; + param->doc= (char *)ftsi.pos; + param->length= ftsi.len; + if (unlikely(parser->parse(param))) + return 0; + } + ftbe=ftb->root; + if (ftbe->docid[1]==docid && ftbe->cur_weight>0 && + ftbe->yesses>=ftbe->ythresh && !ftbe->nos) + { /* row matched ! */ + return ftbe->cur_weight; + } + else + { /* match failed ! */ + return 0.0; + } +} + + +void ft_boolean_close_search(FT_INFO *ftb) +{ + if (is_tree_inited(& ftb->no_dupes)) + { + delete_tree(&ftb->no_dupes, 0); + } + free_root(& ftb->mem_root, MYF(0)); + my_free(ftb); +} + + +float ft_boolean_get_relevance(FT_INFO *ftb) +{ + return ftb->root->cur_weight; +} + + +void ft_boolean_reinit_search(FT_INFO *ftb) +{ + _ftb_init_index_search(ftb); +} + diff --git a/storage/myisam/ft_myisam.c b/storage/myisam/ft_myisam.c new file mode 100644 index 00000000..e88f70ef --- /dev/null +++ b/storage/myisam/ft_myisam.c @@ -0,0 +1,36 @@ +/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +/* + This function is for interface functions between fulltext and myisam +*/ + +#include "ftdefs.h" + +FT_INFO *ft_init_search(uint flags, void *info, uint keynr, + uchar *query, size_t query_len, + CHARSET_INFO *cs, uchar *record) +{ + FT_INFO *res; + if (flags & FT_BOOL) + res= ft_init_boolean_search((MI_INFO *)info, keynr, query, (uint)query_len,cs); + else + res= ft_init_nlq_search((MI_INFO *)info, keynr, query, (uint)query_len, flags, + record); + return res; +} diff --git a/storage/myisam/ft_nlq_search.c b/storage/myisam/ft_nlq_search.c new file mode 100644 index 00000000..90a50905 --- /dev/null +++ b/storage/myisam/ft_nlq_search.c @@ -0,0 +1,386 @@ +/* Copyright (c) 2001, 2011, Oracle and/or its affiliates. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +#define FT_CORE +#include "ftdefs.h" + +/* search with natural language queries */ + +typedef struct ft_doc_rec +{ + my_off_t dpos; + double weight; +} FT_DOC; + +struct st_ft_info +{ + struct _ft_vft *please; + MI_INFO *info; + int ndocs; + int curdoc; + FT_DOC doc[1]; +}; + +typedef struct st_all_in_one +{ + MI_INFO *info; + uint keynr; + CHARSET_INFO *charset; + uchar *keybuff; + TREE dtree; +} ALL_IN_ONE; + +typedef struct st_ft_superdoc +{ + FT_DOC doc; + FT_WORD *word_ptr; + double tmp_weight; +} FT_SUPERDOC; + +static int FT_SUPERDOC_cmp(void* cmp_arg __attribute__((unused)), + FT_SUPERDOC *p1, FT_SUPERDOC *p2) +{ + if (p1->doc.dpos < p2->doc.dpos) + return -1; + if (p1->doc.dpos == p2->doc.dpos) + return 0; + return 1; +} + +static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio) +{ + FT_WEIGTH subkeys; + int r; + uint keylen, doc_cnt; + FT_SUPERDOC sdoc, *sptr; + TREE_ELEMENT *selem; + double gweight=1; + MI_INFO *info=aio->info; + MYISAM_SHARE *share= info->s; + uchar *keybuff=aio->keybuff; + MI_KEYDEF *keyinfo=info->s->keyinfo+aio->keynr; + my_off_t key_root; + uint extra= HA_FT_WLEN + info->s->rec_reflength; + float tmp_weight; + DBUG_ENTER("walk_and_match"); + + word->weight=LWS_FOR_QUERY; + + keylen=_ft_make_key(info,aio->keynr,keybuff,word,0); + keylen-=HA_FT_WLEN; + doc_cnt=0; + subkeys.i= 0; + + if (share->concurrent_insert) + mysql_rwlock_rdlock(&share->key_root_lock[aio->keynr]); + + key_root= share->state.key_root[aio->keynr]; + + /* Skip rows inserted by current inserted */ + for (r=_mi_search(info, keyinfo, keybuff, keylen, SEARCH_FIND, key_root) ; + !r && + (subkeys.i= ft_sintXkorr(info->lastkey+info->lastkey_length-extra)) > 0 && + info->lastpos >= info->state->data_file_length ; + r= _mi_search_next(info, keyinfo, info->lastkey, + info->lastkey_length, SEARCH_BIGGER, key_root)) + ; + + if (share->concurrent_insert) + mysql_rwlock_unlock(&share->key_root_lock[aio->keynr]); + + info->update|= HA_STATE_AKTIV; /* for _mi_test_if_changed() */ + + /* The following should be safe, even if we compare doubles */ + while (!r && gweight) + { + + if (keylen && + ha_compare_word(aio->charset, + info->lastkey + 1, + info->lastkey_length - extra - 1, + keybuff + 1, keylen - 1)) + break; + + if (subkeys.i < 0) + { + if (doc_cnt) + DBUG_RETURN(1); /* index is corrupted */ + /* + TODO here: unsafe optimization, should this word + be skipped (based on subkeys) ? + */ + keybuff+=keylen; + keyinfo=& info->s->ft2_keyinfo; + key_root=info->lastpos; + keylen=0; + if (share->concurrent_insert) + mysql_rwlock_rdlock(&share->key_root_lock[aio->keynr]); + r=_mi_search_first(info, keyinfo, key_root); + goto do_skip; + } + /* The weight we read was actually a float */ + tmp_weight= subkeys.f; + /* The following should be safe, even if we compare doubles */ + if (tmp_weight==0) + DBUG_RETURN(doc_cnt); /* stopword, doc_cnt should be 0 */ + + sdoc.doc.dpos=info->lastpos; + + /* saving document matched into dtree */ + if (!(selem=tree_insert(&aio->dtree, &sdoc, 0, aio->dtree.custom_arg))) + DBUG_RETURN(1); + + sptr=(FT_SUPERDOC *)ELEMENT_KEY((&aio->dtree), selem); + + if (selem->count==1) /* document's first match */ + sptr->doc.weight=0; + else + sptr->doc.weight+=sptr->tmp_weight*sptr->word_ptr->weight; + + sptr->word_ptr=word; + sptr->tmp_weight=tmp_weight; + + doc_cnt++; + + gweight=word->weight*GWS_IN_USE; + if (gweight < 0 || doc_cnt > 2000000) + gweight=0; + + if (share->concurrent_insert) + mysql_rwlock_rdlock(&share->key_root_lock[aio->keynr]); + + if (_mi_test_if_changed(info) == 0) + r=_mi_search_next(info, keyinfo, info->lastkey, info->lastkey_length, + SEARCH_BIGGER, key_root); + else + r=_mi_search(info, keyinfo, info->lastkey, info->lastkey_length, + SEARCH_BIGGER, key_root); +do_skip: + while ((subkeys.i= ft_sintXkorr(info->lastkey+info->lastkey_length-extra)) > 0 && + !r && info->lastpos >= info->state->data_file_length) + r= _mi_search_next(info, keyinfo, info->lastkey, info->lastkey_length, + SEARCH_BIGGER, key_root); + + if (share->concurrent_insert) + mysql_rwlock_unlock(&share->key_root_lock[aio->keynr]); + } + word->weight=gweight; + + DBUG_RETURN(0); +} + + +static int walk_and_copy(FT_SUPERDOC *from, + uint32 count __attribute__((unused)), FT_DOC **to) +{ + DBUG_ENTER("walk_and_copy"); + from->doc.weight+=from->tmp_weight*from->word_ptr->weight; + (*to)->dpos=from->doc.dpos; + (*to)->weight=from->doc.weight; + (*to)++; + DBUG_RETURN(0); +} + +static int walk_and_push(FT_SUPERDOC *from, + uint32 count __attribute__((unused)), QUEUE *best) +{ + DBUG_ENTER("walk_and_copy"); + from->doc.weight+=from->tmp_weight*from->word_ptr->weight; + set_if_smaller(best->elements, ft_query_expansion_limit-1); + queue_insert(best, (uchar *)& from->doc); + DBUG_RETURN(0); +} + + +static int FT_DOC_cmp(void *unused __attribute__((unused)), + FT_DOC *a, FT_DOC *b) +{ + return CMP_NUM(b->weight, a->weight); +} + + +FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, uchar *query, + uint query_len, uint flags, uchar *record) +{ + TREE wtree; + ALL_IN_ONE aio; + FT_DOC *dptr; + FT_INFO *dlist=NULL; + my_off_t saved_lastpos=info->lastpos; + struct st_mysql_ftparser *parser; + MYSQL_FTPARSER_PARAM *ftparser_param; + DBUG_ENTER("ft_init_nlq_search"); + +/* black magic ON */ + if ((int) (keynr = _mi_check_index(info,keynr)) < 0) + DBUG_RETURN(NULL); + if (_mi_readinfo(info,F_RDLCK,1)) + DBUG_RETURN(NULL); +/* black magic OFF */ + + aio.info=info; + aio.keynr=keynr; + aio.charset=info->s->keyinfo[keynr].seg->charset; + aio.keybuff=info->lastkey+info->s->base.max_key_length; + parser= info->s->keyinfo[keynr].parser; + if (! (ftparser_param= ftparser_call_initializer(info, keynr, 0))) + goto err; + + bzero(&wtree,sizeof(wtree)); + + init_tree(&aio.dtree,0,0,sizeof(FT_SUPERDOC),(qsort_cmp2)&FT_SUPERDOC_cmp, + NULL, NULL, MYF(0)); + + ft_parse_init(&wtree, aio.charset); + ftparser_param->flags= 0; + if (ft_parse(&wtree, query, query_len, parser, ftparser_param, + &wtree.mem_root)) + goto err; + + if (tree_walk(&wtree, (tree_walk_action)&walk_and_match, &aio, + left_root_right)) + goto err; + + if (flags & FT_EXPAND && ft_query_expansion_limit) + { + QUEUE best; + init_queue(&best,ft_query_expansion_limit,0,0, (queue_compare) &FT_DOC_cmp, + 0, 0, 0); + tree_walk(&aio.dtree, (tree_walk_action) &walk_and_push, + &best, left_root_right); + while (best.elements) + { + my_off_t docid= ((FT_DOC *)queue_remove_top(&best))->dpos; + if (!(*info->read_record)(info,docid,record)) + { + info->update|= HA_STATE_AKTIV; + ftparser_param->flags= MYSQL_FTFLAGS_NEED_COPY; + if (unlikely(_mi_ft_parse(&wtree, info, keynr, record, ftparser_param, + &wtree.mem_root))) + { + delete_queue(&best); + goto err; + } + } + } + delete_queue(&best); + reset_tree(&aio.dtree); + if (tree_walk(&wtree, (tree_walk_action)&walk_and_match, &aio, + left_root_right)) + goto err; + + } + + /* + If ndocs == 0, this will not allocate RAM for FT_INFO.doc[], + so if ndocs == 0, FT_INFO.doc[] must not be accessed. + */ + dlist=(FT_INFO *)my_malloc(mi_key_memory_FT_INFO, sizeof(FT_INFO)+ + sizeof(FT_DOC)* + (int)(aio.dtree.elements_in_tree-1), + MYF(0)); + if (!dlist) + goto err; + + dlist->please= (struct _ft_vft *) & _ft_vft_nlq; + dlist->ndocs=aio.dtree.elements_in_tree; + dlist->curdoc=-1; + dlist->info=aio.info; + dptr=dlist->doc; + + tree_walk(&aio.dtree, (tree_walk_action) &walk_and_copy, + &dptr, left_root_right); + + if (flags & FT_SORTED) + my_qsort2(dlist->doc, dlist->ndocs, sizeof(FT_DOC), (qsort2_cmp)&FT_DOC_cmp, + 0); + +err: + delete_tree(&aio.dtree, 0); + delete_tree(&wtree, 0); + info->lastpos=saved_lastpos; + DBUG_RETURN(dlist); +} + + +int ft_nlq_read_next(FT_INFO *handler, char *record) +{ + MI_INFO *info= (MI_INFO *) handler->info; + + if (++handler->curdoc >= handler->ndocs) + { + --handler->curdoc; + return HA_ERR_END_OF_FILE; + } + + info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + + info->lastpos=handler->doc[handler->curdoc].dpos; + if (!(*info->read_record)(info,info->lastpos,(uchar*) record)) + { + info->update|= HA_STATE_AKTIV; /* Record is read */ + return 0; + } + return my_errno; +} + + +float ft_nlq_find_relevance(FT_INFO *handler, + uchar *record __attribute__((unused)), + uint length __attribute__((unused))) +{ + int a,b,c; + FT_DOC *docs=handler->doc; + my_off_t docid=handler->info->lastpos; + + if (docid == HA_POS_ERROR) + return -5.0; + + /* Assuming docs[] is sorted by dpos... */ + + for (a=0, b=handler->ndocs, c=(a+b)/2; b-a>1; c=(a+b)/2) + { + if (docs[c].dpos > docid) + b=c; + else + a=c; + } + /* bounds check to avoid accessing unallocated handler->doc */ + if (a < handler->ndocs && docs[a].dpos == docid) + return (float) docs[a].weight; + else + return 0.0; +} + + +void ft_nlq_close_search(FT_INFO *handler) +{ + my_free(handler); +} + + +float ft_nlq_get_relevance(FT_INFO *handler) +{ + return (float) handler->doc[handler->curdoc].weight; +} + + +void ft_nlq_reinit_search(FT_INFO *handler) +{ + handler->curdoc=-1; +} + diff --git a/storage/myisam/ft_parser.c b/storage/myisam/ft_parser.c new file mode 100644 index 00000000..ec392b6e --- /dev/null +++ b/storage/myisam/ft_parser.c @@ -0,0 +1,421 @@ +/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2020, MariaDB Corporation. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +#include "ftdefs.h" + +typedef struct st_ft_docstat { + FT_WORD *list; + uint uniq; + double sum; +} FT_DOCSTAT; + +typedef struct st_my_ft_parser_param +{ + TREE *wtree; + MEM_ROOT *mem_root; +} MY_FT_PARSER_PARAM; + +static int FT_WORD_cmp(CHARSET_INFO* cs, FT_WORD *w1, FT_WORD *w2) +{ + return ha_compare_word(cs, (uchar*) w1->pos, w1->len, + (uchar*) w2->pos, w2->len); +} + +static int walk_and_copy(FT_WORD *word,uint32 count,FT_DOCSTAT *docstat) +{ + word->weight=LWS_IN_USE; + docstat->sum+=word->weight; + memcpy((docstat->list)++, word, sizeof(FT_WORD)); + return 0; +} + +/* transforms tree of words into the array, applying normalization */ + +FT_WORD * ft_linearize(TREE *wtree, MEM_ROOT *mem_root) +{ + FT_WORD *wlist,*p; + FT_DOCSTAT docstat; + DBUG_ENTER("ft_linearize"); + + if ((wlist=(FT_WORD *) alloc_root(mem_root, sizeof(FT_WORD)* + (1+wtree->elements_in_tree)))) + { + docstat.list=wlist; + docstat.uniq=wtree->elements_in_tree; + docstat.sum=0; + tree_walk(wtree,(tree_walk_action)&walk_and_copy,&docstat,left_root_right); + } + delete_tree(wtree, 0); + if (!wlist) + DBUG_RETURN(NULL); + + docstat.list->pos=NULL; + + for (p=wlist;p->pos;p++) + { + p->weight=PRENORM_IN_USE; + } + + for (p=wlist;p->pos;p++) + { + p->weight/=NORM_IN_USE; + } + + DBUG_RETURN(wlist); +} + +my_bool ft_boolean_check_syntax_string(const uchar *str, size_t length, + CHARSET_INFO *cs) +{ + uint i, j; + + if (cs->mbminlen != 1) + { + DBUG_ASSERT(0); + return 1; + } + + if (!str || + (length + 1 != sizeof(DEFAULT_FTB_SYNTAX)) || + (str[0] != ' ' && str[1] != ' ')) + return 1; + for (i=0; i<sizeof(DEFAULT_FTB_SYNTAX); i++) + { + /* limiting to 7-bit ascii only */ + if ((unsigned char)(str[i]) > 127 || my_isalnum(cs, str[i])) + return 1; + for (j=0; j<i; j++) + if (str[i] == str[j] && (i != 11 || j != 10)) + return 1; + } + return 0; +} + +/* + RETURN VALUE + 0 - eof + 1 - word found + 2 - left bracket + 3 - right bracket + 4 - stopword found +*/ +uchar ft_get_word(CHARSET_INFO *cs, const uchar **start, const uchar *end, + FT_WORD *word, MYSQL_FTPARSER_BOOLEAN_INFO *param) +{ + const uchar *doc=*start; + int ctype; + uint mwc, length; + int mbl; + + param->yesno=(FTB_YES==' ') ? 1 : (param->quot != 0); + param->weight_adjust= param->wasign= 0; + param->type= FT_TOKEN_EOF; + + while (doc<end) + { + for (; doc < end; doc+= (mbl > 0 ? mbl : (mbl < 0 ? -mbl : 1))) + { + mbl= my_ci_ctype(cs, &ctype, (uchar*)doc, (uchar*)end); + if (true_word_char(ctype, *doc)) + break; + if (*doc == FTB_RQUOT && param->quot) + { + *start=doc+1; + param->type= FT_TOKEN_RIGHT_PAREN; + goto ret; + } + if (!param->quot) + { + if (*doc == FTB_LBR || *doc == FTB_RBR || *doc == FTB_LQUOT) + { + /* param->prev=' '; */ + *start=doc+1; + if (*doc == FTB_LQUOT) + param->quot= (char*) 1; + param->type= (*doc == FTB_RBR ? FT_TOKEN_RIGHT_PAREN : FT_TOKEN_LEFT_PAREN); + goto ret; + } + if (param->prev == ' ') + { + if (*doc == FTB_YES ) { param->yesno=+1; continue; } else + if (*doc == FTB_EGAL) { param->yesno= 0; continue; } else + if (*doc == FTB_NO ) { param->yesno=-1; continue; } else + if (*doc == FTB_INC ) { param->weight_adjust++; continue; } else + if (*doc == FTB_DEC ) { param->weight_adjust--; continue; } else + if (*doc == FTB_NEG ) { param->wasign= !param->wasign; continue; } + } + } + param->prev=*doc; + param->yesno=(FTB_YES==' ') ? 1 : (param->quot != 0); + param->weight_adjust= param->wasign= 0; + } + + mwc=length=0; + for (word->pos= doc; doc < end; length++, + doc+= (mbl > 0 ? mbl : (mbl < 0 ? -mbl : 1))) + { + mbl= my_ci_ctype(cs, &ctype, (uchar*)doc, (uchar*)end); + if (true_word_char(ctype, *doc)) + mwc=0; + else if (!misc_word_char(*doc) || mwc) + break; + else + mwc++; + } + param->prev='A'; /* be sure *prev is true_word_char */ + word->len= (uint)(doc-word->pos) - mwc; + if ((param->trunc=(doc<end && *doc == FTB_TRUNC))) + doc++; + + if (((length >= ft_min_word_len && !is_stopword((char*) word->pos, + word->len)) + || param->trunc) && length < ft_max_word_len) + { + *start=doc; + param->type= FT_TOKEN_WORD; + goto ret; + } + else if (length) /* make sure length > 0 (if start contains spaces only) */ + { + *start= doc; + param->type= FT_TOKEN_STOPWORD; + goto ret; + } + } + if (param->quot) + { + *start= doc; + param->type= 3; /* FT_RBR */ + goto ret; + } +ret: + return param->type; +} + +uchar ft_simple_get_word(CHARSET_INFO *cs, uchar **start, const uchar *end, + FT_WORD *word, my_bool skip_stopwords) +{ + uchar *doc= *start; + uint mwc, length; + int mbl; + int ctype; + DBUG_ENTER("ft_simple_get_word"); + + do + { + for (;; doc+= (mbl > 0 ? mbl : (mbl < 0 ? -mbl : 1))) + { + if (doc >= end) + DBUG_RETURN(0); + mbl= my_ci_ctype(cs, &ctype, (uchar*)doc, (uchar*)end); + if (true_word_char(ctype, *doc)) + break; + } + + mwc= length= 0; + for (word->pos= doc; doc < end; length++, + doc+= (mbl > 0 ? mbl : (mbl < 0 ? -mbl : 1))) + { + mbl= my_ci_ctype(cs, &ctype, (uchar*)doc, (uchar*)end); + if (true_word_char(ctype, *doc)) + mwc= 0; + else if (!misc_word_char(*doc) || mwc) + break; + else + mwc++; + } + + word->len= (uint)(doc-word->pos) - mwc; + + if (skip_stopwords == FALSE || + (length >= ft_min_word_len && length < ft_max_word_len && + !is_stopword((char*) word->pos, word->len))) + { + *start= doc; + DBUG_RETURN(1); + } + } while (doc < end); + DBUG_RETURN(0); +} + +void ft_parse_init(TREE *wtree, CHARSET_INFO *cs) +{ + DBUG_ENTER("ft_parse_init"); + if (!is_tree_inited(wtree)) + init_tree(wtree, 0, 0, sizeof(FT_WORD), (qsort_cmp2)&FT_WORD_cmp, 0, + (void*)cs, MYF(0)); + DBUG_VOID_RETURN; +} + + +static int ft_add_word(MYSQL_FTPARSER_PARAM *param, + const char *word, int word_len, + MYSQL_FTPARSER_BOOLEAN_INFO *boolean_info __attribute__((unused))) +{ + TREE *wtree; + FT_WORD w; + MY_FT_PARSER_PARAM *ft_param=param->mysql_ftparam; + DBUG_ENTER("ft_add_word"); + wtree= ft_param->wtree; + if (param->flags & MYSQL_FTFLAGS_NEED_COPY) + { + uchar *ptr; + DBUG_ASSERT(wtree->with_delete == 0); + ptr= (uchar *)alloc_root(ft_param->mem_root, word_len); + memcpy(ptr, word, word_len); + w.pos= ptr; + } + else + w.pos= (uchar*) word; + w.len= word_len; + if (!tree_insert(wtree, &w, 0, wtree->custom_arg)) + { + delete_tree(wtree, 0); + DBUG_RETURN(1); + } + DBUG_RETURN(0); +} + + +static int ft_parse_internal(MYSQL_FTPARSER_PARAM *param, + const char *doc_arg, int doc_len) +{ + uchar *doc= (uchar*) doc_arg; + uchar *end= doc + doc_len; + MY_FT_PARSER_PARAM *ft_param=param->mysql_ftparam; + TREE *wtree= ft_param->wtree; + FT_WORD w; + DBUG_ENTER("ft_parse_internal"); + + while (ft_simple_get_word(wtree->custom_arg, &doc, end, &w, TRUE)) + if (param->mysql_add_word(param, (char*) w.pos, (int)w.len, 0)) + DBUG_RETURN(1); + DBUG_RETURN(0); +} + + +int ft_parse(TREE *wtree, uchar *doc, int doclen, + struct st_mysql_ftparser *parser, + MYSQL_FTPARSER_PARAM *param, MEM_ROOT *mem_root) +{ + MY_FT_PARSER_PARAM my_param; + DBUG_ENTER("ft_parse"); + DBUG_ASSERT(parser); + + my_param.wtree= wtree; + my_param.mem_root= mem_root; + + param->mysql_parse= ft_parse_internal; + param->mysql_add_word= ft_add_word; + param->mysql_ftparam= &my_param; + param->cs= wtree->custom_arg; + param->doc= (char*) doc; + param->length= doclen; + param->mode= MYSQL_FTPARSER_SIMPLE_MODE; + DBUG_RETURN(parser->parse(param)); +} + + +#define MAX_PARAM_NR 2 + +MYSQL_FTPARSER_PARAM* ftparser_alloc_param(MI_INFO *info) +{ + if (!info->ftparser_param) + { + /* +. info->ftparser_param can not be zero after the initialization, + because it always includes built-in fulltext parser. And built-in + parser can be called even if the table has no fulltext indexes and + no varchar/text fields. + + ftb_find_relevance... parser (ftb_find_relevance_parse, + ftb_find_relevance_add_word) calls ftb_check_phrase... parser + (ftb_check_phrase_internal, ftb_phrase_add_word). Thus MAX_PARAM_NR=2. + */ + info->ftparser_param= (MYSQL_FTPARSER_PARAM *) + my_malloc(mi_key_memory_FTPARSER_PARAM, + MAX_PARAM_NR * sizeof(MYSQL_FTPARSER_PARAM) * info->s->ftkeys, + MYF(MY_WME | MY_ZEROFILL)); + init_alloc_root(mi_key_memory_ft_memroot, &info->ft_memroot, + FTPARSER_MEMROOT_ALLOC_SIZE, 0, MYF(0)); + } + return info->ftparser_param; +} + + +MYSQL_FTPARSER_PARAM *ftparser_call_initializer(MI_INFO *info, + uint keynr, uint paramnr) +{ + uint32 ftparser_nr; + struct st_mysql_ftparser *parser; + + if (!ftparser_alloc_param(info)) + return 0; + + if (keynr == NO_SUCH_KEY) + { + ftparser_nr= 0; + parser= &ft_default_parser; + } + else + { + ftparser_nr= info->s->keyinfo[keynr].ftkey_nr; + parser= info->s->keyinfo[keynr].parser; + } + DBUG_ASSERT(paramnr < MAX_PARAM_NR); + ftparser_nr= ftparser_nr*MAX_PARAM_NR + paramnr; + if (! info->ftparser_param[ftparser_nr].mysql_add_word) + { + /* Note, that mysql_add_word is used here as a flag: + mysql_add_word == 0 - parser is not initialized + mysql_add_word != 0 - parser is initialized, or no + initialization needed. */ + info->ftparser_param[ftparser_nr].mysql_add_word= + (int (*)(struct st_mysql_ftparser_param *, const char *, int, + MYSQL_FTPARSER_BOOLEAN_INFO *)) 1; + if (parser->init && parser->init(&info->ftparser_param[ftparser_nr])) + return 0; + } + return &info->ftparser_param[ftparser_nr]; +} + +void ftparser_call_deinitializer(MI_INFO *info) +{ + uint i, j, keys= info->s->state.header.keys; + free_root(&info->ft_memroot, MYF(0)); + if (! info->ftparser_param) + return; + for (i= 0; i < keys; i++) + { + MI_KEYDEF *keyinfo= &info->s->keyinfo[i]; + for (j=0; j < MAX_PARAM_NR; j++) + { + MYSQL_FTPARSER_PARAM *ftparser_param= + &info->ftparser_param[keyinfo->ftkey_nr * MAX_PARAM_NR + j]; + if (keyinfo->flag & HA_FULLTEXT && ftparser_param->mysql_add_word) + { + if (keyinfo->parser->deinit) + keyinfo->parser->deinit(ftparser_param); + ftparser_param->mysql_add_word= 0; + } + else + break; + } + } +} + diff --git a/storage/myisam/ft_static.c b/storage/myisam/ft_static.c new file mode 100644 index 00000000..78123cdf --- /dev/null +++ b/storage/myisam/ft_static.c @@ -0,0 +1,624 @@ +/* Copyright (c) 2000-2008 MySQL AB, 2009 Sun Microsystems, Inc. + Use is subject to license terms. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +#include "ftdefs.h" + +ulong ft_min_word_len= 4; +ulong ft_max_word_len= HA_FT_MAXCHARLEN; +ulong ft_query_expansion_limit= 5; +const char *ft_boolean_syntax= DEFAULT_FTB_SYNTAX; + +const HA_KEYSEG ft_keysegs[FT_SEGS]= { +{ + 0, /* charset */ + HA_FT_WLEN, /* start */ + 0, /* null_pos */ + 0, /* Bit pos */ + HA_VAR_LENGTH_PART | HA_PACK_KEY, /* flag */ + HA_FT_MAXBYTELEN, /* length */ + 63, /* language (will be overwritten) */ + HA_KEYTYPE_VARTEXT2, /* type */ + 0, /* null_bit */ + 2, 0 /* bit_start, bit_length */ +}, +{ + /* + Note, this (and the last HA_KEYTYPE_END) segment should NOT + be packed in any way, otherwise w_search() won't be able to + update key entry 'in vivo' + */ + 0, 0, 0, 0, HA_NO_SORT, HA_FT_WLEN, 63, HA_FT_WTYPE, 0, 0, 0 +} +}; + +const struct _ft_vft _ft_vft_nlq= { + ft_nlq_read_next, ft_nlq_find_relevance, ft_nlq_close_search, + ft_nlq_get_relevance, ft_nlq_reinit_search +}; +const struct _ft_vft _ft_vft_boolean= { + ft_boolean_read_next, ft_boolean_find_relevance, ft_boolean_close_search, + ft_boolean_get_relevance, ft_boolean_reinit_search +}; + +const char *ft_stopword_file= 0; +const char *ft_precompiled_stopwords[]= { + +#ifdef COMPILE_STOPWORDS_IN + +/* This particular stopword list was taken from SMART distribution + ftp://ftp.cs.cornell.edu/pub/smart/smart.11.0.tar.Z + it was slightly modified to my taste, though + */ + + "a's", + "able", + "about", + "above", + "according", + "accordingly", + "across", + "actually", + "after", + "afterwards", + "again", + "against", + "ain't", + "all", + "allow", + "allows", + "almost", + "alone", + "along", + "already", + "also", + "although", + "always", + "am", + "among", + "amongst", + "an", + "and", + "another", + "any", + "anybody", + "anyhow", + "anyone", + "anything", + "anyway", + "anyways", + "anywhere", + "apart", + "appear", + "appreciate", + "appropriate", + "are", + "aren't", + "around", + "as", + "aside", + "ask", + "asking", + "associated", + "at", + "available", + "away", + "awfully", + "be", + "became", + "because", + "become", + "becomes", + "becoming", + "been", + "before", + "beforehand", + "behind", + "being", + "believe", + "below", + "beside", + "besides", + "best", + "better", + "between", + "beyond", + "both", + "brief", + "but", + "by", + "c'mon", + "c's", + "came", + "can", + "can't", + "cannot", + "cant", + "cause", + "causes", + "certain", + "certainly", + "changes", + "clearly", + "co", + "com", + "come", + "comes", + "concerning", + "consequently", + "consider", + "considering", + "contain", + "containing", + "contains", + "corresponding", + "could", + "couldn't", + "course", + "currently", + "definitely", + "described", + "despite", + "did", + "didn't", + "different", + "do", + "does", + "doesn't", + "doing", + "don't", + "done", + "down", + "downwards", + "during", + "each", + "edu", + "eg", + "eight", + "either", + "else", + "elsewhere", + "enough", + "entirely", + "especially", + "et", + "etc", + "even", + "ever", + "every", + "everybody", + "everyone", + "everything", + "everywhere", + "ex", + "exactly", + "example", + "except", + "far", + "few", + "fifth", + "first", + "five", + "followed", + "following", + "follows", + "for", + "former", + "formerly", + "forth", + "four", + "from", + "further", + "furthermore", + "get", + "gets", + "getting", + "given", + "gives", + "go", + "goes", + "going", + "gone", + "got", + "gotten", + "greetings", + "had", + "hadn't", + "happens", + "hardly", + "has", + "hasn't", + "have", + "haven't", + "having", + "he", + "he's", + "hello", + "help", + "hence", + "her", + "here", + "here's", + "hereafter", + "hereby", + "herein", + "hereupon", + "hers", + "herself", + "hi", + "him", + "himself", + "his", + "hither", + "hopefully", + "how", + "howbeit", + "however", + "i'd", + "i'll", + "i'm", + "i've", + "ie", + "if", + "ignored", + "immediate", + "in", + "inasmuch", + "inc", + "indeed", + "indicate", + "indicated", + "indicates", + "inner", + "insofar", + "instead", + "into", + "inward", + "is", + "isn't", + "it", + "it'd", + "it'll", + "it's", + "its", + "itself", + "just", + "keep", + "keeps", + "kept", + "know", + "knows", + "known", + "last", + "lately", + "later", + "latter", + "latterly", + "least", + "less", + "lest", + "let", + "let's", + "like", + "liked", + "likely", + "little", + "look", + "looking", + "looks", + "ltd", + "mainly", + "many", + "may", + "maybe", + "me", + "mean", + "meanwhile", + "merely", + "might", + "more", + "moreover", + "most", + "mostly", + "much", + "must", + "my", + "myself", + "name", + "namely", + "nd", + "near", + "nearly", + "necessary", + "need", + "needs", + "neither", + "never", + "nevertheless", + "new", + "next", + "nine", + "no", + "nobody", + "non", + "none", + "noone", + "nor", + "normally", + "not", + "nothing", + "novel", + "now", + "nowhere", + "obviously", + "of", + "off", + "often", + "oh", + "ok", + "okay", + "old", + "on", + "once", + "one", + "ones", + "only", + "onto", + "or", + "other", + "others", + "otherwise", + "ought", + "our", + "ours", + "ourselves", + "out", + "outside", + "over", + "overall", + "own", + "particular", + "particularly", + "per", + "perhaps", + "placed", + "please", + "plus", + "possible", + "presumably", + "probably", + "provides", + "que", + "quite", + "qv", + "rather", + "rd", + "re", + "really", + "reasonably", + "regarding", + "regardless", + "regards", + "relatively", + "respectively", + "right", + "said", + "same", + "saw", + "say", + "saying", + "says", + "second", + "secondly", + "see", + "seeing", + "seem", + "seemed", + "seeming", + "seems", + "seen", + "self", + "selves", + "sensible", + "sent", + "serious", + "seriously", + "seven", + "several", + "shall", + "she", + "should", + "shouldn't", + "since", + "six", + "so", + "some", + "somebody", + "somehow", + "someone", + "something", + "sometime", + "sometimes", + "somewhat", + "somewhere", + "soon", + "sorry", + "specified", + "specify", + "specifying", + "still", + "sub", + "such", + "sup", + "sure", + "t's", + "take", + "taken", + "tell", + "tends", + "th", + "than", + "thank", + "thanks", + "thanx", + "that", + "that's", + "thats", + "the", + "their", + "theirs", + "them", + "themselves", + "then", + "thence", + "there", + "there's", + "thereafter", + "thereby", + "therefore", + "therein", + "theres", + "thereupon", + "these", + "they", + "they'd", + "they'll", + "they're", + "they've", + "think", + "third", + "this", + "thorough", + "thoroughly", + "those", + "though", + "three", + "through", + "throughout", + "thru", + "thus", + "to", + "together", + "too", + "took", + "toward", + "towards", + "tried", + "tries", + "truly", + "try", + "trying", + "twice", + "two", + "un", + "under", + "unfortunately", + "unless", + "unlikely", + "until", + "unto", + "up", + "upon", + "us", + "use", + "used", + "useful", + "uses", + "using", + "usually", + "value", + "various", + "very", + "via", + "viz", + "vs", + "want", + "wants", + "was", + "wasn't", + "way", + "we", + "we'd", + "we'll", + "we're", + "we've", + "welcome", + "well", + "went", + "were", + "weren't", + "what", + "what's", + "whatever", + "when", + "whence", + "whenever", + "where", + "where's", + "whereafter", + "whereas", + "whereby", + "wherein", + "whereupon", + "wherever", + "whether", + "which", + "while", + "whither", + "who", + "who's", + "whoever", + "whole", + "whom", + "whose", + "why", + "will", + "willing", + "wish", + "with", + "within", + "without", + "won't", + "wonder", + "would", + "wouldn't", + "yes", + "yet", + "you", + "you'd", + "you'll", + "you're", + "you've", + "your", + "yours", + "yourself", + "yourselves", + "zero", +#endif + + NULL }; + +static int ft_default_parser_parse(MYSQL_FTPARSER_PARAM *param) +{ + return param->mysql_parse(param, param->doc, param->length); +} + +struct st_mysql_ftparser ft_default_parser= +{ + MYSQL_FTPARSER_INTERFACE_VERSION, ft_default_parser_parse, 0, 0 +}; + diff --git a/storage/myisam/ft_stopwords.c b/storage/myisam/ft_stopwords.c new file mode 100644 index 00000000..b666c1f3 --- /dev/null +++ b/storage/myisam/ft_stopwords.c @@ -0,0 +1,147 @@ +/* Copyright (c) 2000, 2010, Oracle and/or its affiliates + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +#include "ftdefs.h" +#include "my_compare.h" + + +static CHARSET_INFO *ft_stopword_cs= NULL; + + +typedef struct st_ft_stopwords +{ + const char * pos; + uint len; +} FT_STOPWORD; + +static TREE *stopwords3=NULL; + +static int FT_STOPWORD_cmp(void* cmp_arg __attribute__((unused)), + FT_STOPWORD *w1, FT_STOPWORD *w2) +{ + return ha_compare_word(ft_stopword_cs, + (uchar *) w1->pos, w1->len, + (uchar *) w2->pos, w2->len); +} + +static int FT_STOPWORD_free(FT_STOPWORD *w, TREE_FREE action, + void *arg __attribute__((unused))) +{ + if (action == free_free) + my_free((void*)w->pos); + return 0; +} + +static int ft_add_stopword(const char *w) +{ + FT_STOPWORD sw; + return !w || + (((sw.len= (uint) strlen(sw.pos=w)) >= ft_min_word_len) && + (tree_insert(stopwords3, &sw, 0, stopwords3->custom_arg)==NULL)); +} + +int ft_init_stopwords() +{ + DBUG_ENTER("ft_init_stopwords"); + if (!stopwords3) + { + if (!(stopwords3=(TREE *)my_malloc(mi_key_memory_ft_stopwords, + sizeof(TREE), MYF(0)))) + DBUG_RETURN(-1); + init_tree(stopwords3,0,0,sizeof(FT_STOPWORD),(qsort_cmp2)&FT_STOPWORD_cmp, + (ft_stopword_file ? (tree_element_free)&FT_STOPWORD_free : 0), + NULL, MYF(0)); + /* + Stopword engine currently does not support tricky + character sets UCS2, UTF16, UTF32. + Use latin1 to compare stopwords in case of these character sets. + It's also fine to use latin1 with the built-in stopwords. + */ + ft_stopword_cs= default_charset_info->mbminlen == 1 ? + default_charset_info : &my_charset_latin1; + } + + if (ft_stopword_file) + { + File fd; + size_t len; + uchar *buffer, *start, *end; + FT_WORD w; + int error=-1; + + if (!*ft_stopword_file) + DBUG_RETURN(0); + + if ((fd=my_open(ft_stopword_file, O_RDONLY, MYF(MY_WME))) == -1) + DBUG_RETURN(-1); + len=(size_t)my_seek(fd, 0L, MY_SEEK_END, MYF(0)); + my_seek(fd, 0L, MY_SEEK_SET, MYF(0)); + if (!(start= buffer= my_malloc(mi_key_memory_ft_stopwords, len+1, + MYF(MY_WME)))) + goto err0; + len=my_read(fd, buffer, len, MYF(MY_WME)); + end=start+len; + while (ft_simple_get_word(ft_stopword_cs, &start, end, &w, TRUE)) + { + if (ft_add_stopword(my_strndup(mi_key_memory_ft_stopwords, + (char*) w.pos, w.len, MYF(0)))) + goto err1; + } + error=0; +err1: + my_free(buffer); +err0: + my_close(fd, MYF(MY_WME)); + DBUG_RETURN(error); + } + else + { + /* compatibility mode: to be removed */ + char **sws=(char **)ft_precompiled_stopwords; + + for (;*sws;sws++) + { + if (ft_add_stopword(*sws)) + DBUG_RETURN(-1); + } + ft_stopword_file="(built-in)"; /* for SHOW VARIABLES */ + } + DBUG_RETURN(0); +} + +int is_stopword(const char *word, size_t len) +{ + FT_STOPWORD sw; + sw.pos=word; + sw.len=(uint)len; + return tree_search(stopwords3,&sw, stopwords3->custom_arg) != NULL; +} + + +void ft_free_stopwords() +{ + DBUG_ENTER("ft_free_stopwords"); + + if (stopwords3) + { + delete_tree(stopwords3, 0); /* purecov: inspected */ + my_free(stopwords3); + stopwords3=0; + } + ft_stopword_file= 0; + DBUG_VOID_RETURN; +} diff --git a/storage/myisam/ft_update.c b/storage/myisam/ft_update.c new file mode 100644 index 00000000..2da619dc --- /dev/null +++ b/storage/myisam/ft_update.c @@ -0,0 +1,347 @@ +/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +/* functions to work with full-text indices */ + +#include "ftdefs.h" +#include <math.h> + +void _mi_ft_segiterator_init(MI_INFO *info, uint keynr, const uchar *record, + FT_SEG_ITERATOR *ftsi) +{ + DBUG_ENTER("_mi_ft_segiterator_init"); + + ftsi->num=info->s->keyinfo[keynr].keysegs; + ftsi->seg=info->s->keyinfo[keynr].seg; + ftsi->rec=record; + ftsi->pos= 0; /* Avoid warnings from gcc */ + ftsi->len= 0; /* Avoid warnings from gcc */ + DBUG_VOID_RETURN; +} + +void _mi_ft_segiterator_dummy_init(const uchar *record, uint len, + FT_SEG_ITERATOR *ftsi) +{ + DBUG_ENTER("_mi_ft_segiterator_dummy_init"); + + ftsi->num=1; + ftsi->seg=0; + ftsi->pos=record; + ftsi->len=len; + DBUG_VOID_RETURN; +} + +/* + This function breaks convention "return 0 in success" + but it's easier to use like this + + while(_mi_ft_segiterator()) + + so "1" means "OK", "0" means "EOF" +*/ + +uint _mi_ft_segiterator(register FT_SEG_ITERATOR *ftsi) +{ + DBUG_ENTER("_mi_ft_segiterator"); + + if (!ftsi->num) + DBUG_RETURN(0); + + ftsi->num--; + if (!ftsi->seg) + DBUG_RETURN(1); + + ftsi->seg--; + + if (ftsi->seg->null_bit && + (ftsi->rec[ftsi->seg->null_pos] & ftsi->seg->null_bit)) + { + ftsi->pos=0; + DBUG_RETURN(1); + } + ftsi->pos= ftsi->rec+ftsi->seg->start; + if (ftsi->seg->flag & HA_VAR_LENGTH_PART) + { + uint pack_length= (ftsi->seg->bit_start); + ftsi->len= (pack_length == 1 ? (uint) *(uchar*) ftsi->pos : + uint2korr(ftsi->pos)); + ftsi->pos+= pack_length; /* Skip VARCHAR length */ + DBUG_RETURN(1); + } + if (ftsi->seg->flag & HA_BLOB_PART) + { + ftsi->len=_mi_calc_blob_length(ftsi->seg->bit_start,ftsi->pos); + memcpy((char**) &ftsi->pos, ftsi->pos+ftsi->seg->bit_start, sizeof(char*)); + DBUG_RETURN(1); + } + ftsi->len=ftsi->seg->length; + DBUG_RETURN(1); +} + + +/* parses a document i.e. calls ft_parse for every keyseg */ + +uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr, const uchar *record, + MYSQL_FTPARSER_PARAM *param, MEM_ROOT *mem_root) +{ + FT_SEG_ITERATOR ftsi; + struct st_mysql_ftparser *parser; + DBUG_ENTER("_mi_ft_parse"); + + _mi_ft_segiterator_init(info, keynr, record, &ftsi); + + ft_parse_init(parsed, info->s->keyinfo[keynr].seg->charset); + parser= info->s->keyinfo[keynr].parser; + while (_mi_ft_segiterator(&ftsi)) + { + if (ftsi.pos) + if (ft_parse(parsed, (uchar *)ftsi.pos, ftsi.len, parser, param, mem_root)) + DBUG_RETURN(1); + } + DBUG_RETURN(0); +} + +FT_WORD *_mi_ft_parserecord(MI_INFO *info, uint keynr, const uchar *record, + MEM_ROOT *mem_root) +{ + TREE ptree; + MYSQL_FTPARSER_PARAM *param; + DBUG_ENTER("_mi_ft_parserecord"); + if (! (param= ftparser_call_initializer(info, keynr, 0))) + DBUG_RETURN(NULL); + bzero((char*) &ptree, sizeof(ptree)); + param->flags= 0; + if (_mi_ft_parse(&ptree, info, keynr, record, param, mem_root)) + DBUG_RETURN(NULL); + + DBUG_RETURN(ft_linearize(&ptree, mem_root)); +} + +static int _mi_ft_store(MI_INFO *info, uint keynr, uchar *keybuf, + FT_WORD *wlist, my_off_t filepos) +{ + uint key_length; + DBUG_ENTER("_mi_ft_store"); + + for (; wlist->pos; wlist++) + { + key_length=_ft_make_key(info,keynr,keybuf,wlist,filepos); + if (_mi_ck_write(info,keynr,(uchar*) keybuf,key_length)) + DBUG_RETURN(1); + } + DBUG_RETURN(0); +} + +static int _mi_ft_erase(MI_INFO *info, uint keynr, uchar *keybuf, + FT_WORD *wlist, my_off_t filepos) +{ + uint key_length, err=0; + DBUG_ENTER("_mi_ft_erase"); + + for (; wlist->pos; wlist++) + { + key_length=_ft_make_key(info,keynr,keybuf,wlist,filepos); + if (_mi_ck_delete(info,keynr,(uchar*) keybuf,key_length)) + err=1; + } + DBUG_RETURN(err); +} + +/* + Compares an appropriate parts of two WORD_KEY keys directly out of records + returns 1 if they are different +*/ + +#define THOSE_TWO_DAMN_KEYS_ARE_REALLY_DIFFERENT 1 +#define GEE_THEY_ARE_ABSOLUTELY_IDENTICAL 0 + +int _mi_ft_cmp(MI_INFO *info, uint keynr, const uchar *rec1, const uchar *rec2) +{ + FT_SEG_ITERATOR ftsi1, ftsi2; + CHARSET_INFO *cs=info->s->keyinfo[keynr].seg->charset; + DBUG_ENTER("_mi_ft_cmp"); + _mi_ft_segiterator_init(info, keynr, rec1, &ftsi1); + _mi_ft_segiterator_init(info, keynr, rec2, &ftsi2); + + while (_mi_ft_segiterator(&ftsi1) && _mi_ft_segiterator(&ftsi2)) + { + if ((ftsi1.pos != ftsi2.pos) && + (!ftsi1.pos || !ftsi2.pos || + ha_compare_word(cs, (uchar*) ftsi1.pos, ftsi1.len, + (uchar*) ftsi2.pos, ftsi2.len))) + DBUG_RETURN(THOSE_TWO_DAMN_KEYS_ARE_REALLY_DIFFERENT); + } + DBUG_RETURN(GEE_THEY_ARE_ABSOLUTELY_IDENTICAL); +} + + +/* update a document entry */ + +int _mi_ft_update(MI_INFO *info, uint keynr, uchar *keybuf, + const uchar *oldrec, const uchar *newrec, my_off_t pos) +{ + int error= -1; + FT_WORD *oldlist,*newlist, *old_word, *new_word; + CHARSET_INFO *cs=info->s->keyinfo[keynr].seg->charset; + uint key_length; + int cmp, cmp2; + DBUG_ENTER("_mi_ft_update"); + + if (!(old_word=oldlist=_mi_ft_parserecord(info, keynr, oldrec, + &info->ft_memroot)) || + !(new_word=newlist=_mi_ft_parserecord(info, keynr, newrec, + &info->ft_memroot))) + goto err; + + error=0; + while(old_word->pos && new_word->pos) + { + cmp= ha_compare_word(cs, (uchar*) old_word->pos, old_word->len, + (uchar*) new_word->pos, new_word->len); + cmp2= cmp ? 0 : (fabs(old_word->weight - new_word->weight) > 1.e-5); + + if (cmp < 0 || cmp2) + { + key_length=_ft_make_key(info,keynr,keybuf,old_word,pos); + if ((error=_mi_ck_delete(info,keynr,(uchar*) keybuf,key_length))) + goto err; + } + if (cmp > 0 || cmp2) + { + key_length=_ft_make_key(info,keynr,keybuf,new_word,pos); + if ((error=_mi_ck_write(info,keynr,(uchar*) keybuf,key_length))) + goto err; + } + if (cmp<=0) old_word++; + if (cmp>=0) new_word++; + } + if (old_word->pos) + error=_mi_ft_erase(info,keynr,keybuf,old_word,pos); + else if (new_word->pos) + error=_mi_ft_store(info,keynr,keybuf,new_word,pos); + +err: + free_root(&info->ft_memroot, MYF(MY_MARK_BLOCKS_FREE)); + DBUG_RETURN(error); +} + + +/* adds a document to the collection */ + +int _mi_ft_add(MI_INFO *info, uint keynr, uchar *keybuf, const uchar *record, + my_off_t pos) +{ + int error= -1; + FT_WORD *wlist; + DBUG_ENTER("_mi_ft_add"); + DBUG_PRINT("enter",("keynr: %d",keynr)); + + if ((wlist=_mi_ft_parserecord(info, keynr, record, &info->ft_memroot))) + error=_mi_ft_store(info,keynr,keybuf,wlist,pos); + + free_root(&info->ft_memroot, MYF(MY_MARK_BLOCKS_FREE)); + DBUG_PRINT("exit",("Return: %d",error)); + DBUG_RETURN(error); +} + + +/* removes a document from the collection */ + +int _mi_ft_del(MI_INFO *info, uint keynr, uchar *keybuf, const uchar *record, + my_off_t pos) +{ + int error= -1; + FT_WORD *wlist; + DBUG_ENTER("_mi_ft_del"); + DBUG_PRINT("enter",("keynr: %d",keynr)); + + if ((wlist=_mi_ft_parserecord(info, keynr, record, &info->ft_memroot))) + error=_mi_ft_erase(info,keynr,keybuf,wlist,pos); + + free_root(&info->ft_memroot, MYF(MY_MARK_BLOCKS_FREE)); + DBUG_PRINT("exit",("Return: %d",error)); + DBUG_RETURN(error); +} + +uint _ft_make_key(MI_INFO *info, uint keynr, uchar *keybuf, FT_WORD *wptr, + my_off_t filepos) +{ + uchar buf[HA_FT_MAXBYTELEN+16]; + float weight=(float) ((filepos==HA_OFFSET_ERROR) ? 0 : wptr->weight); + DBUG_ENTER("_ft_make_key"); + + mi_float4store(buf,weight); + int2store(buf+HA_FT_WLEN,wptr->len); + memcpy(buf+HA_FT_WLEN+2,wptr->pos,wptr->len); + DBUG_RETURN(_mi_make_key(info,keynr,(uchar*) keybuf,buf,filepos)); +} + + +/* + convert key value to ft2 +*/ + +uint _mi_ft_convert_to_ft2(MI_INFO *info, uint keynr, uchar *key) +{ + my_off_t root; + DYNAMIC_ARRAY *da=info->ft1_to_ft2; + MI_KEYDEF *keyinfo=&info->s->ft2_keyinfo; + uchar *key_ptr= (uchar*) dynamic_array_ptr(da, 0), *end; + size_t length; + uint key_length; + DBUG_ENTER("_mi_ft_convert_to_ft2"); + + /* we'll generate one pageful at once, and insert the rest one-by-one */ + /* calculating the length of this page ...*/ + length=(keyinfo->block_length-2) / keyinfo->keylength; + set_if_smaller(length, da->elements); + length=length * keyinfo->keylength; + + get_key_full_length_rdonly(key_length, key); + while (_mi_ck_delete(info, keynr, key, key_length) == 0) + { + /* + nothing to do here. + _mi_ck_delete() will populate info->ft1_to_ft2 with deleted keys + */ + } + + /* creating pageful of keys */ + mi_putint(info->buff,length+2,0); + memcpy(info->buff+2, key_ptr, length); + info->buff_used=info->page_changed=1; /* info->buff is used */ + if ((root= _mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR || + _mi_write_keypage(info,keyinfo,root,DFLT_INIT_HITS,info->buff)) + DBUG_RETURN(-1); + + /* inserting the rest of key values */ + end= (uchar*) dynamic_array_ptr(da, da->elements); + for (key_ptr+=length; key_ptr < end; key_ptr+=keyinfo->keylength) + if(_mi_ck_real_write_btree(info, keyinfo, key_ptr, 0, &root, SEARCH_SAME)) + DBUG_RETURN(-1); + + /* now, writing the word key entry */ + ft_intXstore(key+key_length, - (int) da->elements); + _mi_dpointer(info, key+key_length+HA_FT_WLEN, root); + + DBUG_RETURN(_mi_ck_real_write_btree(info, + info->s->keyinfo+keynr, + key, 0, + &info->s->state.key_root[keynr], + SEARCH_SAME)); +} + diff --git a/storage/myisam/ftbench/Ecompare.pl b/storage/myisam/ftbench/Ecompare.pl new file mode 100755 index 00000000..a97f126e --- /dev/null +++ b/storage/myisam/ftbench/Ecompare.pl @@ -0,0 +1,112 @@ +#!/usr/bin/env perl + +# Copyright (c) 2003, 2005 MySQL AB +# Use is subject to license terms +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +# compares out-files (as created by Ereport.pl) from dir1/*.out and dir2/*.out +# for each effectiveness column computes the probability of the hypothesis +# "Both files have the same effectiveness" + +# sign test is used to verify that test results are statistically +# significant to support the hypothesis. Function is computed on the fly. + +# basic formula is \sum_{r=0}^R C_N^r 2^{-N} +# As N can be big, we'll work with logarithms +$log2=log(2); +sub probab { + my $N=shift, $R=shift; + + my $r, $sum=0; + + for $r (0..$R) { + $sum+=exp(logfac($N)-logfac($r)-logfac($N-$r)-$N*$log2); + } + return $sum; +} + +# log(N!) +# for N<20 exact value from the table (below) is taken +# otherwise, Stirling approximation for N! is used +sub logfac { + my $n=shift; die "n=$n<0" if $n<0; + return $logfactab[$n] if $n<=$#logfactab; + return $n*log($n)-$n+log(2*3.14159265358*$n)/2; +} +@logfactab=( +0, 0, 0.693147180559945, 1.79175946922805, 3.17805383034795, +4.78749174278205, 6.57925121201010, 8.52516136106541, 10.6046029027453, +12.8018274800815, 15.1044125730755, 17.5023078458739, 19.9872144956619, +22.5521638531234, 25.1912211827387, 27.8992713838409, 30.6718601060807, +33.5050734501369, 36.3954452080331, 39.3398841871995, 42.3356164607535, +); + +############################# main () ############################### +#$p=shift; $m=shift; $p-=$m; +#if($p>$m) { +# print "1 > 2 [+$p-$m]: ", probab($p+$m, $m), "\n"; +#} elsif($p<$m) { +# print "1 < 2 [+$p-$m]: ", probab($p+$m, $p), "\n"; +#} else { +# print "1 = 2 [+$p-$m]: ", probab($p+$m, $m), "\n"; +#} +#exit; + +die "Use: $0 dir1 dir2\n" unless @ARGV==2 && + -d ($dir1=shift) && -d ($dir2=shift); +$_=`cd $dir1; echo *.out`; +s/\.out\b//g; +$total=""; + +for $file (split) { + open(OUT1,$out1="$dir1/$file.out") || die "Cannot open $out1: $!"; + open(OUT2,$out2="$dir2/$file.out") || die "Cannot open $out2: $!"; + + @p=@m=(); + while(!eof(OUT1) || !eof(OUT2)) { + $_=<OUT1>; @l1=split; shift @l1; + $_=<OUT2>; @l2=split; shift @l2; + + die "Number of columns differ in line $.\n" unless $#l1 == $#l2; + + for (0..$#l1) { + $p[$_]+= $l1[$_] > $l2[$_]; + $m[$_]+= $l1[$_] < $l2[$_]; + } + } + + for (0..$#l1) { + $pp[$_]+=$p[$_]; $mm[$_]+=$m[$_]; + $total[$_].=rep($file, ($#l1 ? $_ : undef), $p[$_], $m[$_]); + } + close OUT1; + close OUT2; +} + +for (0..$#l1) { + rep($total[$_], ($#l1 ? $_ : undef), $pp[$_], $mm[$_]); +} + +sub rep { + my ($test, $n, $p, $m, $c, $r)=@_; + + if ($p>$m) { $c=">"; $r="+"; } + elsif($p<$m) { $c="<"; $r="-"; } + else { $c="="; $r="="; } + $n=" $n: " if defined $n; + printf "%-8s $n $dir1 $c $dir2 [+%03d-%03d]: %16.15f\n", + $test, $p, $m, probab($p+$m, ($p>=$m ? $m : $p)); + $r; +} diff --git a/storage/myisam/ftbench/Ecreate.pl b/storage/myisam/ftbench/Ecreate.pl new file mode 100755 index 00000000..78962466 --- /dev/null +++ b/storage/myisam/ftbench/Ecreate.pl @@ -0,0 +1,60 @@ +#!/usr/bin/env perl + +# Copyright (c) 2003, 2005 MySQL AB +# Use is subject to license terms +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +$test=shift || die "Usage $0 testname [option]"; +$option=shift; + +open(D, "<data/$test.d") || die "Cannot open(<data/$test.d): $!"; +open(Q, "<data/$test.q") || die "Cannot open(<data/$test.q): $!"; + +$N=0; + +print <<__HEADER__; +DROP TABLE IF EXISTS $test; +CREATE TABLE $test ( + id int(10) unsigned NOT NULL, + text text NOT NULL, + FULLTEXT KEY text (text) +) TYPE=MyISAM CHARSET=latin1; + +ALTER TABLE $test DISABLE KEYS; +__HEADER__ + +while (<D>) { chomp; + s/'/\\'/g; ++$N; + print "INSERT $test VALUES ($N, '$_');\n"; +} + +print <<__PREP__; +ALTER TABLE $test ENABLE KEYS; +SELECT $N; +__PREP__ + +$N=0; + +while (<Q>) { chomp; + s/'/\\'/g; ++$N; + $_="MATCH text AGAINST ('$_' $option)"; + print "SELECT $N, id, $_ FROM $test WHERE $_;\n"; +} + +print <<__FOOTER__; +DROP TABLE $test; +__FOOTER__ + + diff --git a/storage/myisam/ftbench/Ereport.pl b/storage/myisam/ftbench/Ereport.pl new file mode 100755 index 00000000..a8c7c57e --- /dev/null +++ b/storage/myisam/ftbench/Ereport.pl @@ -0,0 +1,65 @@ +#!/usr/bin/env perl + +# Copyright (c) 2003, 2005 MySQL AB +# Use is subject to license terms +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +die "Use: $0 eval_output qrels_file\n" unless @ARGV==2; + +open(EOUT,$eout=shift) || die "Cannot open $eout: $!"; +open(RELJ,$relj=shift) || die "Cannot open $relj: $!"; + +$_=<EOUT>; +die "$eout must start with a number!\n "unless /^[1-9][0-9]*\n/; +$ndocs=$_+0; + +$qid=0; +$relj_str=<RELJ>; +$eout_str=<EOUT>; + +while(!eof(RELJ) || !eof(EOUT)) { + ++$qid; + %dq=(); + $A=$B=$AB=0; + $Ravg=$Pavg=0; + + while($relj_str =~ /^0*$qid\s+(\d+)/) { + ++$A; + $dq{$1+0}=1; + last unless $relj_str=<RELJ>; + } + # Favg measure = 1/(a/Pavg+(1-a)/Ravg) +sub Favg { my $a=shift; $Pavg*$Ravg ? 1/($a/$Pavg+(1-$a)/$Ravg) : 0; } + # F0 : a=0 -- ignore precision + # F5 : a=0.5 + # F1 : a=1 -- ignore recall + while($eout_str =~ /^$qid\s+(\d+)\s+(\d+(?:\.\d+)?)/) { + $B++; + $AB++ if $dq{$1+0}; + $Ravg+=$AB; + $Pavg+=$AB/$B; + last unless $eout_str=<EOUT>; + } + next unless $A; + + $Ravg/=$B*$A if $B; + $Pavg/=$B if $B; + + printf "%5d %1.12f %1.12f %1.12f\n", $qid, Favg(0),Favg(0.5),Favg(1); +} + +exit 0; + + diff --git a/storage/myisam/ftbench/README b/storage/myisam/ftbench/README new file mode 100644 index 00000000..b1f8b66b --- /dev/null +++ b/storage/myisam/ftbench/README @@ -0,0 +1,43 @@ +1. should be run from myisam/ftbench/ +2. myisam/ftdefs.h should NOT be locked (bk get, not bk edit!) +3. there should be ./data/ subdir with test collections, files: + test1.d + test1.q + test1.r + test2.d + test2.q + test2.r + where test1, test2, etc - are arbitrary test names + + *.[dq] files contain documents/queries one item per line. + + *.r files have the structure: + 1 16 .....blablabla + 1 09 .....blablabla + 2 116 .....blablabla + ... + + that is /^\d+\s+\d+/ + and are sorted by the first number (not necessarily by the second) + +4. there should be ./t/ subdir with test directories + + ./t + ./t/BEST/ + ./t/testdir1/ + ./t/testdir2/ + ... + + there *must* be ./t/BEST/ subdir or a symlink to one of other dirs in ./t + all other names (besides BEST) can be arbitrary + + all test results are compared with BEST results. + + test directories may contain ftdefs.h, my.cnf, ft_mode + (the last one is used as in ... MATCH ... AGAINST ("..." $ft_mode) ...) + NOTE: all *.out files in test directories will NOT be overwritten! + delete them to re-test + +5. run ./ft-test-run.sh +6. go make some coffee + diff --git a/storage/myisam/ftbench/ft-test-run.sh b/storage/myisam/ftbench/ft-test-run.sh new file mode 100755 index 00000000..08294071 --- /dev/null +++ b/storage/myisam/ftbench/ft-test-run.sh @@ -0,0 +1,116 @@ +#!/bin/sh + +# Copyright (c) 2003, 2005, 2006 MySQL AB +# Use is subject to license terms +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Library General Public +# License as published by the Free Software Foundation; version 2 +# of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Library General Public License for more details. +# +# You should have received a copy of the GNU Library General Public +# License along with this library; if not, write to the Free +# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1335 USA + +if [ ! -x ./ft-test-run.sh ] ; then + echo "Usage: ./ft-test-run.sh" + exit 1 +fi + +BASE=`pwd` +DATA=$BASE/var +ROOT=`cd ../..; pwd` +MYSQLD=$ROOT/sql/mysqld +MYSQL=$ROOT/client/mysql +MYSQLADMIN=$ROOT/client/mysqladmin +SOCK=$DATA/mysql.sock +PID=$DATA/mysql.pid +H=../ftdefs.h +OPTS="--no-defaults --socket=$SOCK --character-sets-dir=$ROOT/sql/share/charsets" +DELAY=10 + +stop_myslqd() +{ + [ -S $SOCK ] && $MYSQLADMIN $OPTS shutdown + [ -f $PID ] && kill `cat $PID` && sleep 15 && [ -f $PID ] && kill -9 `cat $PID` +} + +if [ ! -d t/BEST ] ; then + echo "No ./t/BEST directory! Aborting..." + exit 1 +fi +rm -f t/BEST/report.txt +if [ -w $H ] ; then + echo "$H is writeable! Aborting..." + exit 1 +fi + +stop_myslqd +rm -rf var > /dev/null 2>&1 +mkdir var +mkdir var/test + +for batch in t/* ; do + [ ! -d $batch ] && continue + [ $batch -ef t/BEST -a $batch != t/BEST ] && continue + + rm -rf var/test/* > /dev/null 2>&1 + rm -f $H + if [ -f $BASE/$batch/ftdefs.h ] ; then + cat $BASE/$batch/ftdefs.h > $H + chmod a-wx $H + else + bk get -q $H + fi + OPTS="--defaults-file=$BASE/$batch/my.cnf --socket=$SOCK --character-sets-dir=$ROOT/sql/share/charsets" + stop_myslqd + rm -f $MYSQLD + echo "building $batch" + echo "============== $batch ===============" >> var/ft_test.log + (cd $ROOT; gmake) >> var/ft_test.log 2>&1 + + for prog in $MYSQLD $MYSQL $MYSQLADMIN ; do + if [ ! -x $prog ] ; then + echo "build failed: no $prog" + exit 1 + fi + done + + echo "=====================================" >> var/ft_test.log + $MYSQLD $OPTS --basedir=$BASE --pid-file=$PID \ + --language=$ROOT/sql/share/english \ + --skip-grant-tables --skip-innodb \ + --skip-networking --tmpdir=$DATA >> var/ft_test.log 2>&1 & + + sleep $DELAY + $MYSQLADMIN $OPTS ping + if [ $? != 0 ] ; then + echo "$MYSQLD refused to start" + exit 1 + fi + for test in `cd data; echo *.r|sed "s/\.r//g"` ; do + if [ -f $batch/$test.out ] ; then + echo "skipping $batch/$test.out" + continue + fi + echo "testing $batch/$test" + FT_MODE=`cat $batch/ft_mode 2>/dev/null` + ./Ecreate.pl $test "$FT_MODE" | $MYSQL $OPTS --skip-column-names test >var/$test.eval + echo "reporting $batch/$test" + ./Ereport.pl var/$test.eval data/$test.r > $batch/$test.out || exit + done + stop_myslqd + rm -f $H + bk get -q $H + if [ ! $batch -ef t/BEST ] ; then + echo "comparing $batch" + ./Ecompare.pl t/BEST $batch >> t/BEST/report.txt + fi +done + diff --git a/storage/myisam/ftdefs.h b/storage/myisam/ftdefs.h new file mode 100644 index 00000000..d384c7df --- /dev/null +++ b/storage/myisam/ftdefs.h @@ -0,0 +1,156 @@ +/* Copyright (c) 2000-2007 MySQL AB, 2009 Sun Microsystems, Inc. + Use is subject to license terms. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +/* some definitions for full-text indices */ + +/* ftdefs.h is is always included first when used, so we have to include my_global.h here */ +#include <my_global.h> +#include "fulltext.h" +#include <m_ctype.h> +#include <my_tree.h> +#include <queues.h> +#include <mysql/plugin.h> + +#define true_word_char(ctype, character) \ + ((ctype) & (_MY_U | _MY_L | _MY_NMR) || \ + (character) == '_') +#define misc_word_char(X) 0 + +#define FT_MAX_WORD_LEN_FOR_SORT 31 + +#define FTPARSER_MEMROOT_ALLOC_SIZE 65536 + +#define COMPILE_STOPWORDS_IN + +/* Interested readers may consult SMART + (ftp://ftp.cs.cornell.edu/pub/smart/smart.11.0.tar.Z) + for an excellent implementation of vector space model we use. + It also demonstrate the usage of different weghting techniques. + This code, though, is completely original and is not based on the + SMART code but was in some cases inspired by it. + + NORM_PIVOT was taken from the article + A.Singhal, C.Buckley, M.Mitra, "Pivoted Document Length Normalization", + ACM SIGIR'96, 21-29, 1996 + */ + +#define LWS_FOR_QUERY LWS_TF +#define LWS_IN_USE LWS_LOG +#define PRENORM_IN_USE PRENORM_AVG +#define NORM_IN_USE NORM_PIVOT +#define GWS_IN_USE GWS_PROB +/*==============================================================*/ +#define LWS_TF (count) +#define LWS_BINARY (count>0) +#define LWS_SQUARE (count*count) +#define LWS_LOG (count?(log( (double) count)+1):0) +/*--------------------------------------------------------------*/ +#define PRENORM_NONE (p->weight) +#define PRENORM_MAX (p->weight/docstat.max) +#define PRENORM_AUG (0.4+0.6*p->weight/docstat.max) +#define PRENORM_AVG (p->weight/docstat.sum*docstat.uniq) +#define PRENORM_AVGLOG ((1+log(p->weight))/(1+log(docstat.sum/docstat.uniq))) +/*--------------------------------------------------------------*/ +#define NORM_NONE (1) +#define NORM_SUM (docstat.nsum) +#define NORM_COS (sqrt(docstat.nsum2)) + +#define PIVOT_VAL (0.0115) +#define NORM_PIVOT (1+PIVOT_VAL*docstat.uniq) +/*---------------------------------------------------------------*/ +#define GWS_NORM (1/sqrt(sum2)) +#define GWS_GFIDF (sum/doc_cnt) +/* Mysterious, but w/o (double) GWS_IDF performs better :-o */ +#define GWS_IDF log(aio->info->state->records/doc_cnt) +#define GWS_IDF1 log((double)aio->info->state->records/doc_cnt) +#define GWS_PROB ((aio->info->state->records > doc_cnt) ? log(((double)(aio->info->state->records-doc_cnt))/doc_cnt) : 0 ) +#define GWS_FREQ (1.0/doc_cnt) +#define GWS_SQUARED pow(log((double)aio->info->state->records/doc_cnt),2) +#define GWS_CUBIC pow(log((double)aio->info->state->records/doc_cnt),3) +#define GWS_ENTROPY (1-(suml/sum-log(sum))/log(aio->info->state->records)) +/*=================================================================*/ + +/* Boolean search operators */ +#define FTB_YES (ft_boolean_syntax[0]) +#define FTB_EGAL (ft_boolean_syntax[1]) +#define FTB_NO (ft_boolean_syntax[2]) +#define FTB_INC (ft_boolean_syntax[3]) +#define FTB_DEC (ft_boolean_syntax[4]) +#define FTB_LBR (ft_boolean_syntax[5]) +#define FTB_RBR (ft_boolean_syntax[6]) +#define FTB_NEG (ft_boolean_syntax[7]) +#define FTB_TRUNC (ft_boolean_syntax[8]) +#define FTB_LQUOT (ft_boolean_syntax[10]) +#define FTB_RQUOT (ft_boolean_syntax[11]) + +typedef struct st_ft_word { + const uchar *pos; + double weight; + size_t len; +} FT_WORD; + +int is_stopword(const char *word, size_t len); + +uint _ft_make_key(MI_INFO *, uint , uchar *, FT_WORD *, my_off_t); + +uchar ft_get_word(CHARSET_INFO *, const uchar **, const uchar *, FT_WORD *, + MYSQL_FTPARSER_BOOLEAN_INFO *); +uchar ft_simple_get_word(CHARSET_INFO *, uchar **, const uchar *, + FT_WORD *, my_bool); + +typedef struct _st_ft_seg_iterator { + uint num, len; + HA_KEYSEG *seg; + const uchar *rec, *pos; +} FT_SEG_ITERATOR; + +void _mi_ft_segiterator_init(MI_INFO *, uint, const uchar *, FT_SEG_ITERATOR *); +void _mi_ft_segiterator_dummy_init(const uchar *, uint, FT_SEG_ITERATOR *); +uint _mi_ft_segiterator(FT_SEG_ITERATOR *); + +void ft_parse_init(TREE *, CHARSET_INFO *); +int ft_parse(TREE *, uchar *, int, struct st_mysql_ftparser *parser, + MYSQL_FTPARSER_PARAM *, MEM_ROOT *); +FT_WORD * ft_linearize(TREE *, MEM_ROOT *); +FT_WORD * _mi_ft_parserecord(MI_INFO *, uint, const uchar *, MEM_ROOT *); +uint _mi_ft_parse(TREE *, MI_INFO *, uint, const uchar *, + MYSQL_FTPARSER_PARAM *, MEM_ROOT *); + +FT_INFO *ft_init_nlq_search(MI_INFO *, uint, uchar *, uint, uint, uchar *); +FT_INFO *ft_init_boolean_search(MI_INFO *, uint, uchar *, uint, CHARSET_INFO *); + +extern const struct _ft_vft _ft_vft_nlq; +int ft_nlq_read_next(FT_INFO *, char *); +float ft_nlq_find_relevance(FT_INFO *, uchar *, uint); +void ft_nlq_close_search(FT_INFO *); +float ft_nlq_get_relevance(FT_INFO *); +my_off_t ft_nlq_get_docid(FT_INFO *); +void ft_nlq_reinit_search(FT_INFO *); + +extern const struct _ft_vft _ft_vft_boolean; +int ft_boolean_read_next(FT_INFO *, char *); +float ft_boolean_find_relevance(FT_INFO *, uchar *, uint); +void ft_boolean_close_search(FT_INFO *); +float ft_boolean_get_relevance(FT_INFO *); +my_off_t ft_boolean_get_docid(FT_INFO *); +void ft_boolean_reinit_search(FT_INFO *); +MYSQL_FTPARSER_PARAM* ftparser_alloc_param(MI_INFO *info); +extern MYSQL_FTPARSER_PARAM *ftparser_call_initializer(MI_INFO *info, + uint keynr, + uint paramnr); +extern void ftparser_call_deinitializer(MI_INFO *info); diff --git a/storage/myisam/fulltext.h b/storage/myisam/fulltext.h new file mode 100644 index 00000000..adefddbd --- /dev/null +++ b/storage/myisam/fulltext.h @@ -0,0 +1,33 @@ +/* + Copyright (c) 2000, 2010, Oracle and/or its affiliates + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code */ + +/* some definitions for full-text indices */ + +#include "myisamdef.h" +#include "ft_global.h" + +/* If HA_FT_MAXLEN is change to 127 or over, it must be tested properly as + it may cause different representation on disk for full text indexes +*/ +#define HA_FT_MAXLEN 126 + +int _mi_ft_cmp(MI_INFO *, uint, const uchar *, const uchar *); +int _mi_ft_add(MI_INFO *, uint, uchar *, const uchar *, my_off_t); +int _mi_ft_del(MI_INFO *, uint, uchar *, const uchar *, my_off_t); + +uint _mi_ft_convert_to_ft2(MI_INFO *, uint, uchar *); diff --git a/storage/myisam/ha_myisam.cc b/storage/myisam/ha_myisam.cc new file mode 100644 index 00000000..34809f41 --- /dev/null +++ b/storage/myisam/ha_myisam.cc @@ -0,0 +1,2830 @@ +/* + Copyright (c) 2000, 2018, Oracle and/or its affiliates. + Copyright (c) 2009, 2022, MariaDB Corporation. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + + +#ifdef USE_PRAGMA_IMPLEMENTATION +#pragma implementation // gcc: Class implementation +#endif + +#define MYSQL_SERVER 1 +#include <my_global.h> +#include "sql_plugin.h" +#include "myisamdef.h" +#include "sql_priv.h" +#include "key.h" // key_copy +#include <m_ctype.h> +#include <my_bit.h> +#include "ha_myisam.h" +#include "rt_index.h" +#include "sql_table.h" // tablename_to_filename +#include "sql_class.h" // THD +#include "debug_sync.h" +#include "sql_debug.h" + +ulonglong myisam_recover_options; +static ulong opt_myisam_block_size; + +/* bits in myisam_recover_options */ +const char *myisam_recover_names[] = +{ "DEFAULT", "BACKUP", "FORCE", "QUICK", "BACKUP_ALL", "OFF", NullS}; +TYPELIB myisam_recover_typelib= {array_elements(myisam_recover_names)-1,"", + myisam_recover_names, NULL}; + +const char *myisam_stats_method_names[] = {"NULLS_UNEQUAL", "NULLS_EQUAL", + "NULLS_IGNORED", NullS}; +TYPELIB myisam_stats_method_typelib= { + array_elements(myisam_stats_method_names) - 1, "", + myisam_stats_method_names, NULL}; + +static MYSQL_SYSVAR_ULONG(block_size, opt_myisam_block_size, + PLUGIN_VAR_READONLY | PLUGIN_VAR_RQCMDARG, + "Block size to be used for MyISAM index pages", NULL, NULL, + MI_KEY_BLOCK_LENGTH, MI_MIN_KEY_BLOCK_LENGTH, MI_MAX_KEY_BLOCK_LENGTH, + MI_MIN_KEY_BLOCK_LENGTH); + +static MYSQL_SYSVAR_ULONG(data_pointer_size, myisam_data_pointer_size, + PLUGIN_VAR_RQCMDARG, "Default pointer size to be used for MyISAM tables", + NULL, NULL, 6, 2, 7, 1); + +#define MB (1024*1024) +static MYSQL_SYSVAR_ULONGLONG(max_sort_file_size, myisam_max_temp_length, + PLUGIN_VAR_RQCMDARG, "Don't use the fast sort index method to created " + "index if the temporary file would get bigger than this", NULL, NULL, + LONG_MAX/MB*MB, 0, MAX_FILE_SIZE, MB); + +static MYSQL_SYSVAR_SET(recover_options, myisam_recover_options, + PLUGIN_VAR_OPCMDARG|PLUGIN_VAR_READONLY, + "Specifies how corrupted tables should be automatically repaired", + NULL, NULL, HA_RECOVER_BACKUP|HA_RECOVER_QUICK, &myisam_recover_typelib); + +static MYSQL_THDVAR_ULONG(repair_threads, PLUGIN_VAR_RQCMDARG, + "If larger than 1, when repairing a MyISAM table all indexes will be " + "created in parallel, with one thread per index. The value of 1 " + "disables parallel repair", NULL, NULL, + 1, 1, ULONG_MAX, 1); + +static MYSQL_THDVAR_ULONGLONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG, + "The buffer that is allocated when sorting the index when doing " + "a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE", NULL, NULL, + SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX/16, 1); + +static MYSQL_SYSVAR_BOOL(use_mmap, opt_myisam_use_mmap, PLUGIN_VAR_NOCMDARG, + "Use memory mapping for reading and writing MyISAM tables", NULL, NULL, FALSE); + +static MYSQL_SYSVAR_ULONGLONG(mmap_size, myisam_mmap_size, + PLUGIN_VAR_RQCMDARG|PLUGIN_VAR_READONLY, "Restricts the total memory " + "used for memory mapping of MySQL tables", NULL, NULL, + SIZE_T_MAX, MEMMAP_EXTRA_MARGIN, SIZE_T_MAX, 1); + +static MYSQL_THDVAR_ENUM(stats_method, PLUGIN_VAR_RQCMDARG, + "Specifies how MyISAM index statistics collection code should " + "treat NULLs. Possible values of name are NULLS_UNEQUAL (default " + "behavior for 4.1 and later), NULLS_EQUAL (emulate 4.0 behavior), " + "and NULLS_IGNORED", NULL, NULL, + MI_STATS_METHOD_NULLS_NOT_EQUAL, &myisam_stats_method_typelib); + +const LEX_CSTRING MI_CHECK_INFO= { STRING_WITH_LEN("info") }; +const LEX_CSTRING MI_CHECK_WARNING= { STRING_WITH_LEN("warning") }; +const LEX_CSTRING MI_CHECK_ERROR= { STRING_WITH_LEN("error") }; + +#ifndef DBUG_OFF +/** + Causes the thread to wait in a spin lock for a query kill signal. + This function is used by the test frame work to identify race conditions. + + The signal is caught and ignored and the thread is not killed. +*/ + +static void debug_wait_for_kill(const char *info) +{ + DBUG_ENTER("debug_wait_for_kill"); + const char *prev_info; + THD *thd; + thd= current_thd; + prev_info= thd_proc_info(thd, info); + while(!thd->killed) + my_sleep(1000); + DBUG_PRINT("info", ("Exit debug_wait_for_kill")); + thd_proc_info(thd, prev_info); + DBUG_VOID_RETURN; +} + + +class Debug_key_myisam: public Debug_key +{ +public: + Debug_key_myisam() { } + + static void print_keys_myisam(THD *thd, const char *where, + const TABLE *table, + const MI_KEYDEF *keydef, uint count) + { + for (uint i= 0; i < count; i++) + { + Debug_key_myisam tmp; + if (!tmp.append(where, strlen(where)) && + !tmp.append_key(table->s->key_info[i].name, keydef[i].flag)) + tmp.print(thd); + print_keysegs(thd, keydef[i].seg, keydef[i].keysegs); + } + } +}; + +#endif + +/***************************************************************************** +** MyISAM tables +*****************************************************************************/ + +static handler *myisam_create_handler(handlerton *hton, + TABLE_SHARE *table, + MEM_ROOT *mem_root) +{ + return new (mem_root) ha_myisam(hton, table); +} + + +static void mi_check_print(HA_CHECK *param, const LEX_CSTRING* msg_type, + const char *msgbuf) +{ + if (msg_type == &MI_CHECK_INFO) + sql_print_information("%s.%s: %s", param->db_name, param->table_name, + msgbuf); + else if (msg_type == &MI_CHECK_WARNING) + sql_print_warning("%s.%s: %s", param->db_name, param->table_name, + msgbuf); + else + sql_print_error("%s.%s: %s", param->db_name, param->table_name, msgbuf); +} + +// collect errors printed by mi_check routines + +static void mi_check_print_msg(HA_CHECK *param, const LEX_CSTRING *msg_type, + const char *fmt, va_list args) +{ + THD* thd = (THD*)param->thd; + Protocol *protocol= thd->protocol; + size_t length, msg_length; + char msgbuf[MYSQL_ERRMSG_SIZE]; + char name[NAME_LEN*2+2]; + + if (param->testflag & T_SUPPRESS_ERR_HANDLING) + return; + + msg_length= my_vsnprintf(msgbuf, sizeof(msgbuf), fmt, args); + msgbuf[sizeof(msgbuf) - 1] = 0; // healthy paranoia + + DBUG_PRINT(msg_type->str,("message: %s",msgbuf)); + + if (!thd->vio_ok()) + { + mi_check_print(param, msg_type, msgbuf); + return; + } + + if (param->testflag & (T_CREATE_MISSING_KEYS | T_SAFE_REPAIR | + T_AUTO_REPAIR)) + { + myf flag= 0; + if (msg_type == &MI_CHECK_INFO) + flag= ME_NOTE; + else if (msg_type == &MI_CHECK_WARNING) + flag= ME_WARNING; + my_message(ER_NOT_KEYFILE, msgbuf, MYF(flag)); + if (thd->variables.log_warnings > 2 && ! thd->log_all_errors) + mi_check_print(param, msg_type, msgbuf); + return; + } + length=(uint) (strxmov(name, param->db_name,".",param->table_name,NullS) - + name); + /* + TODO: switch from protocol to push_warning here. The main reason we didn't + it yet is parallel repair, which threads have no THD object accessible via + current_thd. + + Also we likely need to lock mutex here (in both cases with protocol and + push_warning). + */ + if (param->need_print_msg_lock) + mysql_mutex_lock(¶m->print_msg_mutex); + + protocol->prepare_for_resend(); + protocol->store(name, length, system_charset_info); + protocol->store(param->op_name, strlen(param->op_name), system_charset_info); + protocol->store(msg_type, system_charset_info); + protocol->store(msgbuf, msg_length, system_charset_info); + if (protocol->write()) + sql_print_error("Failed on my_net_write, writing to stderr instead: %s\n", + msgbuf); + else if (thd->variables.log_warnings > 2) + mi_check_print(param, msg_type, msgbuf); + + if (param->need_print_msg_lock) + mysql_mutex_unlock(¶m->print_msg_mutex); + + return; +} + + +/* + Convert TABLE object to MyISAM key and column definition + + SYNOPSIS + table2myisam() + table_arg in TABLE object. + keydef_out out MyISAM key definition. + recinfo_out out MyISAM column definition. + records_out out Number of fields. + + DESCRIPTION + This function will allocate and initialize MyISAM key and column + definition for further use in mi_create or for a check for underlying + table conformance in merge engine. + + The caller needs to free *recinfo_out after use. Since *recinfo_out + and *keydef_out are allocated with a my_multi_malloc, *keydef_out + is freed automatically when *recinfo_out is freed. + + RETURN VALUE + 0 OK + !0 error code +*/ + +int table2myisam(TABLE *table_arg, MI_KEYDEF **keydef_out, + MI_COLUMNDEF **recinfo_out, uint *records_out) +{ + uint i, j, recpos, minpos, fieldpos, temp_length, length; + enum ha_base_keytype type= HA_KEYTYPE_BINARY; + uchar *record; + KEY *pos; + MI_KEYDEF *keydef; + MI_COLUMNDEF *recinfo, *recinfo_pos; + HA_KEYSEG *keyseg; + TABLE_SHARE *share= table_arg->s; + uint options= share->db_options_in_use; + DBUG_ENTER("table2myisam"); + if (!(my_multi_malloc(PSI_INSTRUMENT_ME, MYF(MY_WME), + recinfo_out, (share->fields * 2 + 2) * sizeof(MI_COLUMNDEF), + keydef_out, share->keys * sizeof(MI_KEYDEF), + &keyseg, + (share->key_parts + share->keys) * sizeof(HA_KEYSEG), + NullS))) + DBUG_RETURN(HA_ERR_OUT_OF_MEM); /* purecov: inspected */ + keydef= *keydef_out; + recinfo= *recinfo_out; + pos= table_arg->key_info; + for (i= 0; i < share->keys; i++, pos++) + { + keydef[i].flag= ((uint16) pos->flags & (HA_NOSAME | HA_FULLTEXT | HA_SPATIAL)); + keydef[i].key_alg= pos->algorithm == HA_KEY_ALG_UNDEF ? + (pos->flags & HA_SPATIAL ? HA_KEY_ALG_RTREE : HA_KEY_ALG_BTREE) : + pos->algorithm; + keydef[i].block_length= pos->block_size; + keydef[i].seg= keyseg; + keydef[i].keysegs= pos->user_defined_key_parts; + for (j= 0; j < pos->user_defined_key_parts; j++) + { + Field *field= pos->key_part[j].field; + type= field->key_type(); + keydef[i].seg[j].flag= pos->key_part[j].key_part_flag; + + if (options & HA_OPTION_PACK_KEYS || + (pos->flags & (HA_PACK_KEY | HA_BINARY_PACK_KEY | + HA_SPACE_PACK_USED))) + { + if (pos->key_part[j].length > 8 && + (type == HA_KEYTYPE_TEXT || + type == HA_KEYTYPE_NUM || + (type == HA_KEYTYPE_BINARY && !field->zero_pack()))) + { + /* No blobs here */ + if (j == 0) + keydef[i].flag|= HA_PACK_KEY; + if (!(field->flags & ZEROFILL_FLAG) && + (field->type() == MYSQL_TYPE_STRING || + field->type() == MYSQL_TYPE_VAR_STRING || + ((int) (pos->key_part[j].length - field->decimals())) >= 4)) + keydef[i].seg[j].flag|= HA_SPACE_PACK; + } + else if (j == 0 && (!(pos->flags & HA_NOSAME) || pos->key_length > 16)) + keydef[i].flag|= HA_BINARY_PACK_KEY; + } + keydef[i].seg[j].type= (int) type; + keydef[i].seg[j].start= pos->key_part[j].offset; + keydef[i].seg[j].length= pos->key_part[j].length; + keydef[i].seg[j].bit_start= keydef[i].seg[j].bit_length= 0; + keydef[i].seg[j].bit_pos= 0; + keydef[i].seg[j].language= field->charset_for_protocol()->number; + + if (field->null_ptr) + { + keydef[i].seg[j].null_bit= field->null_bit; + keydef[i].seg[j].null_pos= (uint) (field->null_ptr- + (uchar*) table_arg->record[0]); + } + else + { + keydef[i].seg[j].null_bit= 0; + keydef[i].seg[j].null_pos= 0; + } + if (field->type() == MYSQL_TYPE_BLOB || + field->type() == MYSQL_TYPE_GEOMETRY) + { + keydef[i].seg[j].flag|= HA_BLOB_PART; + /* save number of bytes used to pack length */ + keydef[i].seg[j].bit_start= (uint) (field->pack_length() - + portable_sizeof_char_ptr); + } + else if (field->type() == MYSQL_TYPE_BIT) + { + keydef[i].seg[j].bit_length= ((Field_bit *) field)->bit_len; + keydef[i].seg[j].bit_start= ((Field_bit *) field)->bit_ofs; + keydef[i].seg[j].bit_pos= (uint) (((Field_bit *) field)->bit_ptr - + (uchar*) table_arg->record[0]); + } + } + keyseg+= pos->user_defined_key_parts; + } + if (table_arg->found_next_number_field) + keydef[share->next_number_index].flag|= HA_AUTO_KEY; + record= table_arg->record[0]; + recpos= 0; + recinfo_pos= recinfo; + while (recpos < (uint) share->stored_rec_length) + { + Field **field, *found= 0; + minpos= share->reclength; + length= 0; + + for (field= table_arg->field; *field; field++) + { + if ((fieldpos= (*field)->offset(record)) >= recpos && + fieldpos <= minpos) + { + /* skip null fields */ + if (!(temp_length= (*field)->pack_length_in_rec())) + continue; /* Skip null-fields */ + if (! found || fieldpos < minpos || + (fieldpos == minpos && temp_length < length)) + { + minpos= fieldpos; + found= *field; + length= temp_length; + } + } + } + DBUG_PRINT("loop", ("found: %p recpos: %d minpos: %d length: %d", + found, recpos, minpos, length)); + if (recpos != minpos) + { + /* reserve space for null bits */ + bzero((char*) recinfo_pos, sizeof(*recinfo_pos)); + recinfo_pos->type= FIELD_NORMAL; + recinfo_pos++->length= (uint16) (minpos - recpos); + } + if (!found) + break; + + if (found->flags & BLOB_FLAG) + recinfo_pos->type= FIELD_BLOB; + else if (found->real_type() == MYSQL_TYPE_TIMESTAMP) + { + /* pre-MySQL-5.6.4 TIMESTAMP, or MariaDB-5.3+ TIMESTAMP */ + recinfo_pos->type= FIELD_NORMAL; + } + else if (found->type() == MYSQL_TYPE_VARCHAR) + recinfo_pos->type= FIELD_VARCHAR; + else if (!(options & HA_OPTION_PACK_RECORD)) + recinfo_pos->type= FIELD_NORMAL; + else if (found->real_type() == MYSQL_TYPE_TIMESTAMP2) + { + /* + MySQL-5.6.4+ erroneously marks Field_timestampf as FIELD_SKIP_PRESPACE, + but only if HA_OPTION_PACK_RECORD is set. + */ + recinfo_pos->type= FIELD_SKIP_PRESPACE; + } + else if (found->zero_pack()) + recinfo_pos->type= FIELD_SKIP_ZERO; + else + recinfo_pos->type= ((length <= 3 || + (found->flags & ZEROFILL_FLAG)) ? + FIELD_NORMAL : + found->type() == MYSQL_TYPE_STRING || + found->type() == MYSQL_TYPE_VAR_STRING ? + FIELD_SKIP_ENDSPACE : + FIELD_SKIP_PRESPACE); + if (found->null_ptr) + { + recinfo_pos->null_bit= found->null_bit; + recinfo_pos->null_pos= (uint) (found->null_ptr - + (uchar*) table_arg->record[0]); + } + else + { + recinfo_pos->null_bit= 0; + recinfo_pos->null_pos= 0; + } + (recinfo_pos++)->length= (uint16) length; + recpos= minpos + length; + DBUG_PRINT("loop", ("length: %d type: %d", + recinfo_pos[-1].length,recinfo_pos[-1].type)); + } + *records_out= (uint) (recinfo_pos - recinfo); + DBUG_RETURN(0); +} + + +/* + Check for underlying table conformance + + SYNOPSIS + myisam_check_definition() + t1_keyinfo in First table key definition + t1_recinfo in First table record definition + t1_keys in Number of keys in first table + t1_recs in Number of records in first table + t2_keyinfo in Second table key definition + t2_recinfo in Second table record definition + t2_keys in Number of keys in second table + t2_recs in Number of records in second table + strict in Strict check switch + table in handle to the table object + + DESCRIPTION + This function compares two MyISAM definitions. By intention it was done + to compare merge table definition against underlying table definition. + It may also be used to compare dot-frm and MYI definitions of MyISAM + table as well to compare different MyISAM table definitions. + + For merge table it is not required that number of keys in merge table + must exactly match number of keys in underlying table. When calling this + function for underlying table conformance check, 'strict' flag must be + set to false, and converted merge definition must be passed as t1_*. + + Otherwise 'strict' flag must be set to 1 and it is not required to pass + converted dot-frm definition as t1_*. + + For compatibility reasons we relax some checks, specifically: + - 4.0 (and earlier versions) always set key_alg to 0. + - 4.0 (and earlier versions) have the same language for all keysegs. + + RETURN VALUE + 0 - Equal definitions. + 1 - Different definitions. + + TODO + - compare FULLTEXT keys; + - compare SPATIAL keys; + - compare FIELD_SKIP_ZERO which is converted to FIELD_NORMAL correctly + (should be corretly detected in table2myisam). +*/ + +int check_definition(MI_KEYDEF *t1_keyinfo, MI_COLUMNDEF *t1_recinfo, + uint t1_keys, uint t1_recs, + MI_KEYDEF *t2_keyinfo, MI_COLUMNDEF *t2_recinfo, + uint t2_keys, uint t2_recs, bool strict, TABLE *table_arg) +{ + uint i, j; + DBUG_ENTER("check_definition"); + my_bool mysql_40_compat= table_arg && table_arg->s->frm_version < FRM_VER_TRUE_VARCHAR; + if ((strict ? t1_keys != t2_keys : t1_keys > t2_keys)) + { + DBUG_PRINT("error", ("Number of keys differs: t1_keys=%u, t2_keys=%u", + t1_keys, t2_keys)); + DBUG_RETURN(1); + } + if (t1_recs != t2_recs) + { + DBUG_PRINT("error", ("Number of recs differs: t1_recs=%u, t2_recs=%u", + t1_recs, t2_recs)); + DBUG_RETURN(1); + } + for (i= 0; i < t1_keys; i++) + { + HA_KEYSEG *t1_keysegs= t1_keyinfo[i].seg; + HA_KEYSEG *t2_keysegs= t2_keyinfo[i].seg; + if (t1_keyinfo[i].flag & HA_FULLTEXT && t2_keyinfo[i].flag & HA_FULLTEXT) + continue; + else if (t1_keyinfo[i].flag & HA_FULLTEXT || + t2_keyinfo[i].flag & HA_FULLTEXT) + { + DBUG_PRINT("error", ("Key %d has different definition", i)); + DBUG_PRINT("error", ("t1_fulltext= %d, t2_fulltext=%d", + MY_TEST(t1_keyinfo[i].flag & HA_FULLTEXT), + MY_TEST(t2_keyinfo[i].flag & HA_FULLTEXT))); + DBUG_RETURN(1); + } + if (t1_keyinfo[i].flag & HA_SPATIAL && t2_keyinfo[i].flag & HA_SPATIAL) + continue; + else if (t1_keyinfo[i].flag & HA_SPATIAL || + t2_keyinfo[i].flag & HA_SPATIAL) + { + DBUG_PRINT("error", ("Key %d has different definition", i)); + DBUG_PRINT("error", ("t1_spatial= %d, t2_spatial=%d", + MY_TEST(t1_keyinfo[i].flag & HA_SPATIAL), + MY_TEST(t2_keyinfo[i].flag & HA_SPATIAL))); + DBUG_RETURN(1); + } + if ((!mysql_40_compat && + t1_keyinfo[i].key_alg != t2_keyinfo[i].key_alg) || + t1_keyinfo[i].keysegs != t2_keyinfo[i].keysegs) + { + DBUG_PRINT("error", ("Key %d has different definition", i)); + DBUG_PRINT("error", ("t1_keysegs=%d, t1_key_alg=%d", + t1_keyinfo[i].keysegs, t1_keyinfo[i].key_alg)); + DBUG_PRINT("error", ("t2_keysegs=%d, t2_key_alg=%d", + t2_keyinfo[i].keysegs, t2_keyinfo[i].key_alg)); + DBUG_RETURN(1); + } + for (j= t1_keyinfo[i].keysegs; j--;) + { + uint8 t1_keysegs_j__type= t1_keysegs[j].type; + + /* + Table migration from 4.1 to 5.1. In 5.1 a *TEXT key part is + always HA_KEYTYPE_VARTEXT2. In 4.1 we had only the equivalent of + HA_KEYTYPE_VARTEXT1. Since we treat both the same on MyISAM + level, we can ignore a mismatch between these types. + */ + if ((t1_keysegs[j].flag & HA_BLOB_PART) && + (t2_keysegs[j].flag & HA_BLOB_PART)) + { + if ((t1_keysegs_j__type == HA_KEYTYPE_VARTEXT2) && + (t2_keysegs[j].type == HA_KEYTYPE_VARTEXT1)) + t1_keysegs_j__type= HA_KEYTYPE_VARTEXT1; /* purecov: tested */ + else if ((t1_keysegs_j__type == HA_KEYTYPE_VARBINARY2) && + (t2_keysegs[j].type == HA_KEYTYPE_VARBINARY1)) + t1_keysegs_j__type= HA_KEYTYPE_VARBINARY1; /* purecov: inspected */ + } + + if ((!mysql_40_compat && + t1_keysegs[j].language != t2_keysegs[j].language) || + t1_keysegs_j__type != t2_keysegs[j].type || + t1_keysegs[j].null_bit != t2_keysegs[j].null_bit || + t1_keysegs[j].length != t2_keysegs[j].length || + t1_keysegs[j].start != t2_keysegs[j].start || + (t1_keysegs[j].flag ^ t2_keysegs[j].flag) & HA_REVERSE_SORT) + { + DBUG_PRINT("error", ("Key segment %d (key %d) has different " + "definition", j, i)); + DBUG_PRINT("error", ("t1_type=%d, t1_language=%d, t1_null_bit=%d, " + "t1_length=%d", + t1_keysegs[j].type, t1_keysegs[j].language, + t1_keysegs[j].null_bit, t1_keysegs[j].length)); + DBUG_PRINT("error", ("t2_type=%d, t2_language=%d, t2_null_bit=%d, " + "t2_length=%d", + t2_keysegs[j].type, t2_keysegs[j].language, + t2_keysegs[j].null_bit, t2_keysegs[j].length)); + + DBUG_RETURN(1); + } + } + } + for (i= 0; i < t1_recs; i++) + { + MI_COLUMNDEF *t1_rec= &t1_recinfo[i]; + MI_COLUMNDEF *t2_rec= &t2_recinfo[i]; + /* + FIELD_SKIP_ZERO can be changed to FIELD_NORMAL in mi_create, + see NOTE1 in mi_create.c + */ + if ((t1_rec->type != t2_rec->type && + !(t1_rec->type == (int) FIELD_SKIP_ZERO && + t1_rec->length == 1 && + t2_rec->type == (int) FIELD_NORMAL)) || + t1_rec->length != t2_rec->length || + t1_rec->null_bit != t2_rec->null_bit) + { + DBUG_PRINT("error", ("Field %d has different definition", i)); + DBUG_PRINT("error", ("t1_type=%d, t1_length=%d, t1_null_bit=%d", + t1_rec->type, t1_rec->length, t1_rec->null_bit)); + DBUG_PRINT("error", ("t2_type=%d, t2_length=%d, t2_null_bit=%d", + t2_rec->type, t2_rec->length, t2_rec->null_bit)); + DBUG_RETURN(1); + } + } + DBUG_RETURN(0); +} + +extern "C" { + +int killed_ptr(HA_CHECK *param) +{ + if (likely(thd_killed((THD*)param->thd)) == 0) + return 0; + my_errno= HA_ERR_ABORTED_BY_USER; + return 1; +} + +void mi_check_print_error(HA_CHECK *param, const char *fmt,...) +{ + param->error_printed|=1; + param->out_flag|= O_DATA_LOST; + if (param->testflag & T_SUPPRESS_ERR_HANDLING) + return; + va_list args; + va_start(args, fmt); + mi_check_print_msg(param, &MI_CHECK_ERROR, fmt, args); + va_end(args); +} + +void mi_check_print_info(HA_CHECK *param, const char *fmt,...) +{ + va_list args; + va_start(args, fmt); + mi_check_print_msg(param, &MI_CHECK_INFO, fmt, args); + param->note_printed= 1; + va_end(args); +} + +void mi_check_print_warning(HA_CHECK *param, const char *fmt,...) +{ + param->warning_printed=1; + param->out_flag|= O_DATA_LOST; + va_list args; + va_start(args, fmt); + mi_check_print_msg(param, &MI_CHECK_WARNING, fmt, args); + va_end(args); +} + + +/** + Report list of threads (and queries) accessing a table, thread_id of a + thread that detected corruption, ource file name and line number where + this corruption was detected, optional extra information (string). + + This function is intended to be used when table corruption is detected. + + @param[in] file MI_INFO object. + @param[in] message Optional error message. + @param[in] sfile Name of source file. + @param[in] sline Line number in source file. + + @return void +*/ + +void _mi_report_crashed(MI_INFO *file, const char *message, + const char *sfile, uint sline) +{ + THD *cur_thd; + LIST *element; + char buf[1024]; + mysql_mutex_lock(&file->s->intern_lock); + if ((cur_thd= (THD*) file->in_use.data)) + sql_print_error("Got an error from thread_id=%lld, %s:%d", cur_thd->thread_id, + sfile, sline); + else + sql_print_error("Got an error from unknown thread, %s:%d", sfile, sline); + if (message) + sql_print_error("%s", message); + for (element= file->s->in_use; element; element= list_rest(element)) + { + THD *thd= (THD*) element->data; + sql_print_error("%s", + thd ? + thd_get_error_context_description(thd, buf, sizeof(buf), 0) + : "Unknown thread accessing table"); + } + mysql_mutex_unlock(&file->s->intern_lock); +} + +/* Return 1 if user have requested query to be killed */ + +my_bool mi_killed_in_mariadb(MI_INFO *info) +{ + return (((TABLE*) (info->external_ref))->in_use->killed != 0); +} + +static int compute_vcols(MI_INFO *info, uchar *record, int keynum) +{ + /* This mutex is needed for parallel repair */ + mysql_mutex_lock(&info->s->intern_lock); + TABLE *table= (TABLE*)(info->external_ref); + table->move_fields(table->field, record, table->field[0]->record_ptr()); + if (keynum == -1) // update all vcols + { + int error= table->update_virtual_fields(table->file, VCOL_UPDATE_FOR_READ); + if (table->update_virtual_fields(table->file, VCOL_UPDATE_INDEXED)) + error= 1; + mysql_mutex_unlock(&info->s->intern_lock); + return error; + } + // update only one key + KEY *key= table->key_info + keynum; + KEY_PART_INFO *kp= key->key_part, *end= kp + key->ext_key_parts; + for (; kp < end; kp++) + { + Field *f= table->field[kp->fieldnr - 1]; + if (f->vcol_info && !f->vcol_info->stored_in_db) + table->update_virtual_field(f, false); + } + mysql_mutex_unlock(&info->s->intern_lock); + return 0; +} + +} + +ha_myisam::ha_myisam(handlerton *hton, TABLE_SHARE *table_arg) + :handler(hton, table_arg), file(0), + int_table_flags(HA_NULL_IN_KEY | HA_CAN_FULLTEXT | HA_CAN_SQL_HANDLER | + HA_BINLOG_ROW_CAPABLE | HA_BINLOG_STMT_CAPABLE | + HA_CAN_VIRTUAL_COLUMNS | HA_CAN_EXPORT | + HA_REQUIRES_KEY_COLUMNS_FOR_DELETE | + HA_DUPLICATE_POS | HA_CAN_INDEX_BLOBS | HA_AUTO_PART_KEY | + HA_FILE_BASED | HA_CAN_GEOMETRY | HA_NO_TRANSACTIONS | + HA_CAN_INSERT_DELAYED | HA_CAN_BIT_FIELD | HA_CAN_RTREEKEYS | + HA_HAS_RECORDS | HA_STATS_RECORDS_IS_EXACT | HA_CAN_REPAIR | + HA_CAN_TABLES_WITHOUT_ROLLBACK), + can_enable_indexes(0) +{} + +handler *ha_myisam::clone(const char *name __attribute__((unused)), + MEM_ROOT *mem_root) +{ + ha_myisam *new_handler= + static_cast <ha_myisam *>(handler::clone(file->filename, mem_root)); + if (new_handler) + new_handler->file->state= file->state; + return new_handler; +} + + +static const char *ha_myisam_exts[] = { + ".MYI", + ".MYD", + NullS +}; + +const char *ha_myisam::index_type(uint key_number) +{ + return ((table->key_info[key_number].flags & HA_FULLTEXT) ? + "FULLTEXT" : + (table->key_info[key_number].flags & HA_SPATIAL) ? + "SPATIAL" : + (table->key_info[key_number].algorithm == HA_KEY_ALG_RTREE) ? + "RTREE" : + "BTREE"); +} + + +ulong ha_myisam::index_flags(uint inx, uint part, bool all_parts) const +{ + ulong flags; + if (table_share->key_info[inx].algorithm == HA_KEY_ALG_FULLTEXT) + flags= 0; + else + if ((table_share->key_info[inx].flags & HA_SPATIAL || + table_share->key_info[inx].algorithm == HA_KEY_ALG_RTREE)) + { + /* All GIS scans are non-ROR scans. We also disable IndexConditionPushdown */ + flags= HA_READ_NEXT | HA_READ_PREV | HA_READ_RANGE | + HA_READ_ORDER | HA_KEYREAD_ONLY | HA_KEY_SCAN_NOT_ROR; + } + else + { + flags= HA_READ_NEXT | HA_READ_PREV | HA_READ_RANGE | + HA_READ_ORDER | HA_KEYREAD_ONLY | HA_DO_INDEX_COND_PUSHDOWN | + HA_DO_RANGE_FILTER_PUSHDOWN; + } + return flags; +} + + +/* Name is here without an extension */ +int ha_myisam::open(const char *name, int mode, uint test_if_locked) +{ + MI_KEYDEF *keyinfo; + MI_COLUMNDEF *recinfo= 0; + char readlink_buf[FN_REFLEN], name_buff[FN_REFLEN]; + uint recs; + uint i; + + /* + If the user wants to have memory mapped data files, add an + open_flag. Do not memory map temporary tables because they are + expected to be inserted and thus extended a lot. Memory mapping is + efficient for files that keep their size, but very inefficient for + growing files. Using an open_flag instead of calling mi_extra(... + HA_EXTRA_MMAP ...) after mi_open() has the advantage that the + mapping is not repeated for every open, but just done on the initial + open, when the MyISAM share is created. Every time the server + requires opening a new instance of a table it calls this method. We + will always supply HA_OPEN_MMAP for a permanent table. However, the + MyISAM storage engine will ignore this flag if this is a secondary + open of a table that is in use by other threads already (if the + MyISAM share exists already). + */ + if (!(test_if_locked & HA_OPEN_TMP_TABLE) && opt_myisam_use_mmap) + test_if_locked|= HA_OPEN_MMAP; + + if (!(file=mi_open(name, mode, test_if_locked | HA_OPEN_FROM_SQL_LAYER))) + return (my_errno ? my_errno : -1); + + file->s->chst_invalidator= query_cache_invalidate_by_MyISAM_filename_ref; + /* Set external_ref, mainly for temporary tables */ + file->external_ref= (void*) table; // For mi_killed() + + /* No need to perform a check for tmp table or if it's already checked */ + if (!table->s->tmp_table && file->s->reopen == 1) + { + if ((my_errno= table2myisam(table, &keyinfo, &recinfo, &recs))) + { + /* purecov: begin inspected */ + DBUG_PRINT("error", ("Failed to convert TABLE object to MyISAM " + "key and column definition")); + goto err; + /* purecov: end */ + } + if (check_definition(keyinfo, recinfo, table->s->keys, recs, + file->s->keyinfo, file->s->rec, + file->s->base.keys, file->s->base.fields, + true, table)) + { + /* purecov: begin inspected */ + my_errno= HA_ERR_INCOMPATIBLE_DEFINITION; + goto err; + /* purecov: end */ + } + } + + DBUG_EXECUTE_IF("key", + Debug_key_myisam::print_keys_myisam(table->in_use, + "ha_myisam::open: ", + table, file->s->keyinfo, + file->s->base.keys); + ); + + if (test_if_locked & (HA_OPEN_IGNORE_IF_LOCKED | HA_OPEN_TMP_TABLE)) + (void) mi_extra(file, HA_EXTRA_NO_WAIT_LOCK, 0); + + info(HA_STATUS_NO_LOCK | HA_STATUS_VARIABLE | HA_STATUS_CONST); + + /* + Set data_file_name and index_file_name to point at the symlink value + if table is symlinked (Ie; Real name is not same as generated name) + */ + fn_format(name_buff, file->filename, "", MI_NAME_DEXT, + MY_APPEND_EXT | MY_UNPACK_FILENAME); + if (my_is_symlink(name_buff)) + { + my_readlink(readlink_buf, name_buff, MYF(0)); + data_file_name= strdup_root(&table->mem_root, readlink_buf); + } + else + data_file_name= 0; + fn_format(name_buff, file->filename, "", MI_NAME_IEXT, + MY_APPEND_EXT | MY_UNPACK_FILENAME); + if (my_is_symlink(name_buff)) + { + my_readlink(readlink_buf, name_buff, MYF(0)); + index_file_name= strdup_root(&table->mem_root, readlink_buf); + } + else + index_file_name= 0; + + if (!(test_if_locked & HA_OPEN_WAIT_IF_LOCKED)) + (void) mi_extra(file, HA_EXTRA_WAIT_LOCK, 0); + if (!table->s->db_record_offset) + int_table_flags|=HA_REC_NOT_IN_SEQ; + if (file->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD)) + { + /* + Set which type of automatic checksum we have + The old checksum and new checksum are identical if there is no + null fields. + Files with new checksum has the HA_OPTION_NULL_FIELDS bit set. + */ + if ((file->s->options & HA_OPTION_NULL_FIELDS) || + !file->s->has_null_fields) + int_table_flags|= HA_HAS_NEW_CHECKSUM; + if (!(file->s->options & HA_OPTION_NULL_FIELDS)) + int_table_flags|= HA_HAS_OLD_CHECKSUM; + } + + /* + For static size rows, tell MariaDB that we will access all bytes + in the record when writing it. This signals MariaDB to initialize + the full row to ensure we don't get any errors from valgrind and + that all bytes in the row is properly reset. + */ + if (!(file->s->options & + (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) && + (file->s->has_varchar_fields || file->s->has_null_fields)) + int_table_flags|= HA_RECORD_MUST_BE_CLEAN_ON_WRITE; + + for (i= 0; i < table->s->keys; i++) + { + plugin_ref parser= table->key_info[i].parser; + if (table->key_info[i].flags & HA_USES_PARSER) + file->s->keyinfo[i].parser= + (struct st_mysql_ftparser *)plugin_decl(parser)->info; + table->key_info[i].block_size= file->s->keyinfo[i].block_length; + } + my_errno= 0; + + /* Count statistics of usage for newly open normal files */ + if (file->s->reopen == 1 && ! (test_if_locked & HA_OPEN_TMP_TABLE)) + { + /* use delay_key_write from .frm, not .MYI */ + file->s->delay_key_write= delay_key_write_options == DELAY_KEY_WRITE_ALL || + (delay_key_write_options == DELAY_KEY_WRITE_ON && + table->s->db_create_options & HA_OPTION_DELAY_KEY_WRITE); + if (file->s->delay_key_write) + feature_files_opened_with_delayed_keys++; + } + goto end; + + err: + this->close(); + end: + /* + Both recinfo and keydef are allocated by my_multi_malloc(), thus only + recinfo must be freed. + */ + if (recinfo) + my_free(recinfo); + return my_errno; +} + +int ha_myisam::close(void) +{ + MI_INFO *tmp=file; + if (!tmp) + return 0; + file=0; + return mi_close(tmp); +} + +int ha_myisam::write_row(const uchar *buf) +{ + /* + If we have an auto_increment column and we are writing a changed row + or a new row, then update the auto_increment value in the record. + */ + if (table->next_number_field && buf == table->record[0]) + { + int error; + if ((error= update_auto_increment())) + return error; + } + return mi_write(file,buf); +} + +void ha_myisam::setup_vcols_for_repair(HA_CHECK *param) +{ + DBUG_ASSERT(file->s->base.reclength <= file->s->vreclength); + if (!table->vfield) + return; + + if (file->s->base.reclength == file->s->vreclength) + { + bool indexed_vcols= false; + ulong new_vreclength= file->s->vreclength; + for (Field **vf= table->vfield; *vf; vf++) + { + if (!(*vf)->stored_in_db()) + { + uint vf_end= ((*vf)->offset(table->record[0]) + + (*vf)->pack_length_in_rec()); + set_if_bigger(new_vreclength, vf_end); + indexed_vcols|= ((*vf)->flags & PART_KEY_FLAG) != 0; + } + } + if (!indexed_vcols) + return; + file->s->vreclength= new_vreclength; + } + DBUG_ASSERT(file->s->base.reclength < file->s->vreclength || + !table->s->stored_fields); + param->fix_record= compute_vcols; + table->use_all_columns(); +} + +void ha_myisam::restore_vcos_after_repair() +{ + if (file->s->base.reclength < file->s->vreclength) + { + table->move_fields(table->field, table->record[0], + table->field[0]->record_ptr()); + table->default_column_bitmaps(); + } +} + +int ha_myisam::check(THD* thd, HA_CHECK_OPT* check_opt) +{ + if (!file) return HA_ADMIN_INTERNAL_ERROR; + int error; + HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); + MYISAM_SHARE* share = file->s; + const char *old_proc_info=thd->proc_info; + + if (!param) + return HA_ADMIN_INTERNAL_ERROR; + + thd_proc_info(thd, "Checking table"); + myisamchk_init(param); + param->thd = thd; + param->op_name = "check"; + param->db_name= table->s->db.str; + param->table_name= table->alias.c_ptr(); + param->testflag = check_opt->flags | T_CHECK | T_SILENT; + param->stats_method= (enum_handler_stats_method)THDVAR(thd, stats_method); + + if (!(table->db_stat & HA_READ_ONLY)) + param->testflag|= T_STATISTICS; + param->using_global_keycache = 1; + + if (!mi_is_crashed(file) && + (((param->testflag & T_CHECK_ONLY_CHANGED) && + !(share->state.changed & (STATE_CHANGED | STATE_CRASHED | + STATE_CRASHED_ON_REPAIR)) && + share->state.open_count == 0) || + ((param->testflag & T_FAST) && (share->state.open_count == + (uint) (share->global_changed ? 1 : 0))))) + return HA_ADMIN_ALREADY_DONE; + + setup_vcols_for_repair(param); + + error = chk_status(param, file); // Not fatal + error = chk_size(param, file); + if (!error) + error |= chk_del(param, file, param->testflag); + if (!error) + error = chk_key(param, file); + if (!error) + { + if ((!(param->testflag & T_QUICK) && + ((share->options & + (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) || + (param->testflag & (T_EXTEND | T_MEDIUM)))) || + mi_is_crashed(file)) + { + ulonglong old_testflag= param->testflag; + param->testflag|=T_MEDIUM; + if (!(error= init_io_cache(¶m->read_cache, file->dfile, + my_default_record_cache_size, READ_CACHE, + share->pack.header_length, 1, MYF(MY_WME)))) + { + error= chk_data_link(param, file, MY_TEST(param->testflag & T_EXTEND)); + end_io_cache(¶m->read_cache); + } + param->testflag= old_testflag; + } + } + if (!error) + { + if ((share->state.changed & (STATE_CHANGED | + STATE_CRASHED_ON_REPAIR | + STATE_CRASHED | STATE_NOT_ANALYZED)) || + (param->testflag & T_STATISTICS) || + mi_is_crashed(file)) + { + file->update|=HA_STATE_CHANGED | HA_STATE_ROW_CHANGED; + mysql_mutex_lock(&share->intern_lock); + share->state.changed&= ~(STATE_CHANGED | STATE_CRASHED | + STATE_CRASHED_ON_REPAIR); + if (!(table->db_stat & HA_READ_ONLY)) + error=update_state_info(param,file,UPDATE_TIME | UPDATE_OPEN_COUNT | + UPDATE_STAT); + mysql_mutex_unlock(&share->intern_lock); + info(HA_STATUS_NO_LOCK | HA_STATUS_TIME | HA_STATUS_VARIABLE | + HA_STATUS_CONST); + /* + Write a 'table is ok' message to error log if table is ok and + we have written to error log that table was getting checked + */ + if (!error && !(table->db_stat & HA_READ_ONLY) && + !mi_is_crashed(file) && thd->error_printed_to_log && + (param->warning_printed || param->error_printed || + param->note_printed)) + mi_check_print_info(param, "Table is fixed"); + } + } + else if (!mi_is_crashed(file) && !thd->killed) + { + mi_mark_crashed(file); + file->update |= HA_STATE_CHANGED | HA_STATE_ROW_CHANGED; + } + + restore_vcos_after_repair(); + + thd_proc_info(thd, old_proc_info); + return error ? HA_ADMIN_CORRUPT : HA_ADMIN_OK; +} + + +/* + analyze the key distribution in the table + As the table may be only locked for read, we have to take into account that + two threads may do an analyze at the same time! +*/ + +int ha_myisam::analyze(THD *thd, HA_CHECK_OPT* check_opt) +{ + int error=0; + HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); + MYISAM_SHARE* share = file->s; + + if (!param) + return HA_ADMIN_INTERNAL_ERROR; + + myisamchk_init(param); + param->thd = thd; + param->op_name= "analyze"; + param->db_name= table->s->db.str; + param->table_name= table->alias.c_ptr(); + param->testflag= (T_FAST | T_CHECK | T_SILENT | T_STATISTICS | + T_DONT_CHECK_CHECKSUM); + param->using_global_keycache = 1; + param->stats_method= (enum_handler_stats_method)THDVAR(thd, stats_method); + + if (!(share->state.changed & STATE_NOT_ANALYZED)) + return HA_ADMIN_ALREADY_DONE; + + setup_vcols_for_repair(param); + + error = chk_key(param, file); + if (!error) + { + mysql_mutex_lock(&share->intern_lock); + error=update_state_info(param,file,UPDATE_STAT); + mysql_mutex_unlock(&share->intern_lock); + } + else if (!mi_is_crashed(file) && !thd->killed) + mi_mark_crashed(file); + + restore_vcos_after_repair(); + + return error ? HA_ADMIN_CORRUPT : HA_ADMIN_OK; +} + + +int ha_myisam::repair(THD* thd, HA_CHECK_OPT *check_opt) +{ + int error; + HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); + ha_rows start_records; + + if (!file || !param) return HA_ADMIN_INTERNAL_ERROR; + + myisamchk_init(param); + param->thd = thd; + param->op_name= "repair"; + param->testflag= ((check_opt->flags & ~(T_EXTEND)) | + T_SILENT | T_FORCE_CREATE | T_CALC_CHECKSUM | + (check_opt->flags & T_EXTEND ? T_REP : T_REP_BY_SORT)); + param->tmpfile_createflag= O_RDWR | O_TRUNC; + param->sort_buffer_length= THDVAR(thd, sort_buffer_size); + param->backup_time= check_opt->start_time; + start_records=file->state->records; + + setup_vcols_for_repair(param); + + while ((error=repair(thd,*param,0)) && param->retry_repair) + { + param->retry_repair=0; + if (test_all_bits(param->testflag, + (uint) (T_RETRY_WITHOUT_QUICK | T_QUICK))) + { + param->testflag&= ~(T_RETRY_WITHOUT_QUICK | T_QUICK); + /* Ensure we don't loose any rows when retrying without quick */ + param->testflag|= T_SAFE_REPAIR; + sql_print_information("Retrying repair of: '%s' including modifying data file", + table->s->path.str); + continue; + } + param->testflag&= ~T_QUICK; + if ((param->testflag & (T_REP_BY_SORT | T_REP_PARALLEL))) + { + param->testflag= (param->testflag & ~T_REP_ANY) | T_REP; + sql_print_information("Retrying repair of: '%s' with keycache", + table->s->path.str); + continue; + } + break; + } + + restore_vcos_after_repair(); + + if (!error && start_records != file->state->records && + !(check_opt->flags & T_VERY_SILENT)) + { + char llbuff[22],llbuff2[22]; + sql_print_information("Found %s of %s rows when repairing '%s'", + llstr(file->state->records, llbuff), + llstr(start_records, llbuff2), + table->s->path.str); + } + return error; +} + +int ha_myisam::optimize(THD* thd, HA_CHECK_OPT *check_opt) +{ + int error; + HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); + + if (!file || !param) return HA_ADMIN_INTERNAL_ERROR; + + myisamchk_init(param); + param->thd = thd; + param->op_name= "optimize"; + param->testflag= (check_opt->flags | T_SILENT | T_FORCE_CREATE | + T_REP_BY_SORT | T_STATISTICS | T_SORT_INDEX); + param->tmpfile_createflag= O_RDWR | O_TRUNC; + param->sort_buffer_length= THDVAR(thd, sort_buffer_size); + + setup_vcols_for_repair(param); + + if ((error= repair(thd,*param,1)) && param->retry_repair) + { + sql_print_warning("Warning: Optimize table got errno %d on %s.%s, retrying", + my_errno, param->db_name, param->table_name); + param->testflag&= ~T_REP_BY_SORT; + error= repair(thd,*param,1); + } + + restore_vcos_after_repair(); + + return error; +} + + +int ha_myisam::repair(THD *thd, HA_CHECK ¶m, bool do_optimize) +{ + int error=0; + ulonglong local_testflag= param.testflag; + bool optimize_done= !do_optimize, statistics_done=0; + const char *old_proc_info=thd->proc_info; + char fixed_name[FN_REFLEN]; + MYISAM_SHARE* share = file->s; + ha_rows rows= file->state->records; + my_bool locking= 0; + DBUG_ENTER("ha_myisam::repair"); + + param.db_name= table->s->db.str; + param.table_name= table->alias.c_ptr(); + param.using_global_keycache = 1; + param.thd= thd; + param.tmpdir= &mysql_tmpdir_list; + param.out_flag= 0; + share->state.dupp_key= MI_MAX_KEY; + strmov(fixed_name,file->filename); + + /* + Don't lock tables if we have used LOCK TABLE or if we come from + enable_index() + */ + if (!thd->locked_tables_mode && ! (param.testflag & T_NO_LOCKS)) + { + locking= 1; + if (mi_lock_database(file, table->s->tmp_table ? F_EXTRA_LCK : F_WRLCK)) + { + mi_check_print_error(¶m, ER_THD(thd, ER_CANT_LOCK), my_errno); + DBUG_RETURN(HA_ADMIN_FAILED); + } + } + + if (!do_optimize || + ((file->state->del || share->state.split != file->state->records) && + (!(param.testflag & T_QUICK) || + !(share->state.changed & STATE_NOT_OPTIMIZED_KEYS)))) + { + ulonglong tmp_key_map= ((local_testflag & T_CREATE_MISSING_KEYS) ? + mi_get_mask_all_keys_active(share->base.keys) : + share->state.key_map); + ulonglong testflag= param.testflag; +#ifdef HAVE_MMAP + bool remap= MY_TEST(share->file_map); + /* + mi_repair*() functions family use file I/O even if memory + mapping is available. + + Since mixing mmap I/O and file I/O may cause various artifacts, + memory mapping must be disabled. + */ + if (remap) + mi_munmap_file(file); +#endif + /* + The following is to catch errors when my_errno is no set properly + during repairt + */ + my_errno= 0; + if (mi_test_if_sort_rep(file,file->state->records,tmp_key_map,0) && + (local_testflag & T_REP_BY_SORT)) + { + local_testflag|= T_STATISTICS; + param.testflag|= T_STATISTICS; // We get this for free + statistics_done=1; + if (THDVAR(thd, repair_threads)>1) + { + /* TODO: respect myisam_repair_threads variable */ + thd_proc_info(thd, "Parallel repair"); + error = mi_repair_parallel(¶m, file, fixed_name, + MY_TEST(param.testflag & T_QUICK)); + } + else + { + thd_proc_info(thd, "Repair by sorting"); + DEBUG_SYNC(thd, "myisam_before_repair_by_sort"); + error = mi_repair_by_sort(¶m, file, fixed_name, + MY_TEST(param.testflag & T_QUICK)); + } + if (error && file->create_unique_index_by_sort && + share->state.dupp_key != MAX_KEY) + { + my_errno= HA_ERR_FOUND_DUPP_KEY; + print_keydup_error(table, &table->key_info[share->state.dupp_key], + MYF(0)); + } + } + else + { + thd_proc_info(thd, "Repair with keycache"); + param.testflag &= ~T_REP_BY_SORT; + error= mi_repair(¶m, file, fixed_name, + MY_TEST(param.testflag & T_QUICK)); + } + param.testflag= testflag | (param.testflag & T_RETRY_WITHOUT_QUICK); +#ifdef HAVE_MMAP + if (remap) + mi_dynmap_file(file, file->state->data_file_length); +#endif + optimize_done=1; + } + if (!error) + { + if ((local_testflag & T_SORT_INDEX) && + (share->state.changed & STATE_NOT_SORTED_PAGES)) + { + optimize_done=1; + thd_proc_info(thd, "Sorting index"); + error=mi_sort_index(¶m,file,fixed_name); + } + if (!error && !statistics_done && (local_testflag & T_STATISTICS)) + { + if (share->state.changed & STATE_NOT_ANALYZED) + { + optimize_done=1; + thd_proc_info(thd, "Analyzing"); + error = chk_key(¶m, file); + } + else + local_testflag&= ~T_STATISTICS; // Don't update statistics + } + } + thd_proc_info(thd, "Saving state"); + if (!error) + { + if ((share->state.changed & STATE_CHANGED) || mi_is_crashed(file)) + { + share->state.changed&= ~(STATE_CHANGED | STATE_CRASHED | + STATE_CRASHED_ON_REPAIR); + file->update|=HA_STATE_CHANGED | HA_STATE_ROW_CHANGED; + } + /* + the following 'if', thought conceptually wrong, + is a useful optimization nevertheless. + */ + if (file->state != &file->s->state.state) + file->s->state.state = *file->state; + if (file->s->base.auto_key) + update_auto_increment_key(¶m, file, 1); + if (optimize_done) + { + mysql_mutex_lock(&share->intern_lock); + error = update_state_info(¶m, file, + UPDATE_TIME | UPDATE_OPEN_COUNT | + (local_testflag & + T_STATISTICS ? UPDATE_STAT : 0)); + mysql_mutex_unlock(&share->intern_lock); + } + info(HA_STATUS_NO_LOCK | HA_STATUS_TIME | HA_STATUS_VARIABLE | + HA_STATUS_CONST); + if (rows != file->state->records && ! (param.testflag & T_VERY_SILENT)) + { + char llbuff[22],llbuff2[22]; + mi_check_print_warning(¶m,"Number of rows changed from %s to %s", + llstr(rows,llbuff), + llstr(file->state->records,llbuff2)); + } + } + else + { + mi_mark_crashed_on_repair(file); + file->update |= HA_STATE_CHANGED | HA_STATE_ROW_CHANGED; + update_state_info(¶m, file, 0); + } + thd_proc_info(thd, old_proc_info); + if (locking) + mi_lock_database(file,F_UNLCK); + DBUG_RETURN(error ? HA_ADMIN_FAILED : + !optimize_done ? HA_ADMIN_ALREADY_DONE : HA_ADMIN_OK); +} + + +/* + Assign table indexes to a specific key cache. +*/ + +int ha_myisam::assign_to_keycache(THD* thd, HA_CHECK_OPT *check_opt) +{ + KEY_CACHE *new_key_cache= check_opt->key_cache; + const char *errmsg= 0; + char buf[STRING_BUFFER_USUAL_SIZE]; + int error= HA_ADMIN_OK; + ulonglong map; + TABLE_LIST *table_list= table->pos_in_table_list; + DBUG_ENTER("ha_myisam::assign_to_keycache"); + + table->keys_in_use_for_query.clear_all(); + + if (table_list->process_index_hints(table)) + DBUG_RETURN(HA_ADMIN_FAILED); + map= ~(ulonglong) 0; + if (!table->keys_in_use_for_query.is_clear_all()) + /* use all keys if there's no list specified by the user through hints */ + map= table->keys_in_use_for_query.to_ulonglong(); + + if ((error= mi_assign_to_key_cache(file, map, new_key_cache))) + { + my_snprintf(buf, sizeof(buf), + "Failed to flush to index file (errno: %d)", error); + errmsg= buf; + error= HA_ADMIN_CORRUPT; + } + + if (error != HA_ADMIN_OK) + { + /* Send error to user */ + HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); + if (!param) + return HA_ADMIN_INTERNAL_ERROR; + + myisamchk_init(param); + param->thd= thd; + param->op_name= "assign_to_keycache"; + param->db_name= table->s->db.str; + param->table_name= table->s->table_name.str; + param->testflag= 0; + mi_check_print_error(param, errmsg); + } + DBUG_RETURN(error); +} + + +/* + Preload pages of the index file for a table into the key cache. +*/ + +int ha_myisam::preload_keys(THD* thd, HA_CHECK_OPT *check_opt) +{ + int error; + const char *errmsg; + ulonglong map; + TABLE_LIST *table_list= table->pos_in_table_list; + my_bool ignore_leaves= table_list->ignore_leaves; + char buf[MYSQL_ERRMSG_SIZE]; + + DBUG_ENTER("ha_myisam::preload_keys"); + + table->keys_in_use_for_query.clear_all(); + + if (table_list->process_index_hints(table)) + DBUG_RETURN(HA_ADMIN_FAILED); + + map= ~(ulonglong) 0; + /* Check validity of the index references */ + if (!table->keys_in_use_for_query.is_clear_all()) + /* use all keys if there's no list specified by the user through hints */ + map= table->keys_in_use_for_query.to_ulonglong(); + + mi_extra(file, HA_EXTRA_PRELOAD_BUFFER_SIZE, + (void *) &thd->variables.preload_buff_size); + + if ((error= mi_preload(file, map, ignore_leaves))) + { + switch (error) { + case HA_ERR_NON_UNIQUE_BLOCK_SIZE: + errmsg= "Indexes use different block sizes"; + break; + case HA_ERR_OUT_OF_MEM: + errmsg= "Failed to allocate buffer"; + break; + default: + my_snprintf(buf, sizeof(buf), + "Failed to read from index file (errno: %d)", my_errno); + errmsg= buf; + } + error= HA_ADMIN_FAILED; + goto err; + } + + DBUG_RETURN(HA_ADMIN_OK); + + err: + { + HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); + if (!param) + return HA_ADMIN_INTERNAL_ERROR; + myisamchk_init(param); + param->thd= thd; + param->op_name= "preload_keys"; + param->db_name= table->s->db.str; + param->table_name= table->s->table_name.str; + param->testflag= 0; + mi_check_print_error(param, errmsg); + DBUG_RETURN(error); + } +} + + +/* + Disable indexes, making it persistent if requested. + + SYNOPSIS + disable_indexes() + mode mode of operation: + HA_KEY_SWITCH_NONUNIQ disable all non-unique keys + HA_KEY_SWITCH_ALL disable all keys + HA_KEY_SWITCH_NONUNIQ_SAVE dis. non-uni. and make persistent + HA_KEY_SWITCH_ALL_SAVE dis. all keys and make persistent + + IMPLEMENTATION + HA_KEY_SWITCH_NONUNIQ is not implemented. + HA_KEY_SWITCH_ALL_SAVE is not implemented. + + RETURN + 0 ok + HA_ERR_WRONG_COMMAND mode not implemented. +*/ + +int ha_myisam::disable_indexes(uint mode) +{ + int error; + + if (mode == HA_KEY_SWITCH_ALL) + { + /* call a storage engine function to switch the key map */ + error= mi_disable_indexes(file); + } + else if (mode == HA_KEY_SWITCH_NONUNIQ_SAVE) + { + mi_extra(file, HA_EXTRA_NO_KEYS, 0); + info(HA_STATUS_CONST); // Read new key info + error= 0; + } + else + { + /* mode not implemented */ + error= HA_ERR_WRONG_COMMAND; + } + return error; +} + + +/* + Enable indexes, making it persistent if requested. + + SYNOPSIS + enable_indexes() + mode mode of operation: + HA_KEY_SWITCH_NONUNIQ enable all non-unique keys + HA_KEY_SWITCH_ALL enable all keys + HA_KEY_SWITCH_NONUNIQ_SAVE en. non-uni. and make persistent + HA_KEY_SWITCH_ALL_SAVE en. all keys and make persistent + + DESCRIPTION + Enable indexes, which might have been disabled by disable_index() before. + The modes without _SAVE work only if both data and indexes are empty, + since the MyISAM repair would enable them persistently. + To be sure in these cases, call handler::delete_all_rows() before. + + IMPLEMENTATION + HA_KEY_SWITCH_NONUNIQ is not implemented. + HA_KEY_SWITCH_ALL_SAVE is not implemented. + + RETURN + 0 ok + !=0 Error, among others: + HA_ERR_CRASHED data or index is non-empty. Delete all rows and retry. + HA_ERR_WRONG_COMMAND mode not implemented. +*/ + +int ha_myisam::enable_indexes(uint mode) +{ + int error; + DBUG_ENTER("ha_myisam::enable_indexes"); + + DBUG_EXECUTE_IF("wait_in_enable_indexes", + debug_wait_for_kill("wait_in_enable_indexes"); ); + + if (mi_is_all_keys_active(file->s->state.key_map, file->s->base.keys)) + { + /* All indexes are enabled already. */ + DBUG_RETURN(0); + } + + if (mode == HA_KEY_SWITCH_ALL) + { + error= mi_enable_indexes(file); + /* + Do not try to repair on error, + as this could make the enabled state persistent, + but mode==HA_KEY_SWITCH_ALL forbids it. + */ + } + else if (mode == HA_KEY_SWITCH_NONUNIQ_SAVE) + { + THD *thd= table->in_use; + int was_error= thd->is_error(); + HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); + const char *save_proc_info=thd->proc_info; + + if (!param) + DBUG_RETURN(HA_ADMIN_INTERNAL_ERROR); + + thd_proc_info(thd, "Creating index"); + myisamchk_init(param); + param->op_name= "recreating_index"; + param->testflag= (T_SILENT | T_REP_BY_SORT | T_QUICK | + T_CREATE_MISSING_KEYS); + /* + Don't lock and unlock table if it's locked. + Normally table should be locked. This test is mostly for safety. + */ + if (likely(file->lock_type != F_UNLCK)) + param->testflag|= T_NO_LOCKS; + + if (file->create_unique_index_by_sort) + param->testflag|= T_CREATE_UNIQUE_BY_SORT; + + param->myf_rw&= ~MY_WAIT_IF_FULL; + param->sort_buffer_length= THDVAR(thd, sort_buffer_size); + param->stats_method= (enum_handler_stats_method)THDVAR(thd, stats_method); + param->tmpdir=&mysql_tmpdir_list; + + setup_vcols_for_repair(param); + + if ((error= (repair(thd,*param,0) != HA_ADMIN_OK)) && param->retry_repair) + { + sql_print_warning("Warning: Enabling keys got errno %d on %s.%s, retrying", + my_errno, param->db_name, param->table_name); + /* + Repairing by sort failed. Now try standard repair method. + Still we want to fix only index file. If data file corruption + was detected (T_RETRY_WITHOUT_QUICK), we shouldn't do much here. + Let implicit repair do this job. + */ + if (!(param->testflag & T_RETRY_WITHOUT_QUICK)) + { + param->testflag&= ~T_REP_BY_SORT; + error= (repair(thd,*param,0) != HA_ADMIN_OK); + } + /* + If the standard repair succeeded, clear all error messages which + might have been set by the first repair. They can still be seen + with SHOW WARNINGS then. + */ + if (! error && ! was_error) + thd->clear_error(); + } + info(HA_STATUS_CONST); + thd_proc_info(thd, save_proc_info); + + restore_vcos_after_repair(); + } + else + { + /* mode not implemented */ + error= HA_ERR_WRONG_COMMAND; + } + DBUG_RETURN(error); +} + + +/* + Test if indexes are disabled. + + + SYNOPSIS + indexes_are_disabled() + no parameters + + + RETURN + 0 indexes are not disabled + 1 all indexes are disabled + [2 non-unique indexes are disabled - NOT YET IMPLEMENTED] +*/ + +int ha_myisam::indexes_are_disabled(void) +{ + + return mi_indexes_are_disabled(file); +} + + +/* + prepare for a many-rows insert operation + e.g. - disable indexes (if they can be recreated fast) or + activate special bulk-insert optimizations + + SYNOPSIS + start_bulk_insert(rows, flags) + rows Rows to be inserted + 0 if we don't know + flags Flags to control index creation + + NOTICE + Do not forget to call end_bulk_insert() later! +*/ + +void ha_myisam::start_bulk_insert(ha_rows rows, uint flags) +{ + DBUG_ENTER("ha_myisam::start_bulk_insert"); + THD *thd= table->in_use; + ulong size= MY_MIN(thd->variables.read_buff_size, + (ulong) (table->s->avg_row_length*rows)); + bool index_disabled= 0; + DBUG_PRINT("info",("start_bulk_insert: rows %lu size %lu", + (ulong) rows, size)); + + /* don't enable row cache if too few rows */ + if ((!rows || rows > MI_MIN_ROWS_TO_USE_WRITE_CACHE) && !has_long_unique()) + mi_extra(file, HA_EXTRA_WRITE_CACHE, (void*) &size); + + can_enable_indexes= mi_is_all_keys_active(file->s->state.key_map, + file->s->base.keys); + + /* + Only disable old index if the table was empty and we are inserting + a lot of rows. + Note that in end_bulk_insert() we may truncate the table if + enable_indexes() failed, thus it's essential that indexes are + disabled ONLY for an empty table. + */ + if (file->state->records == 0 && can_enable_indexes && + (!rows || rows >= MI_MIN_ROWS_TO_DISABLE_INDEXES)) + { + if (file->open_flag & HA_OPEN_INTERNAL_TABLE) + { + file->update|= HA_STATE_CHANGED; + index_disabled= file->s->base.keys > 0; + mi_clear_all_keys_active(file->s->state.key_map); + } + else + { + my_bool all_keys= MY_TEST(flags & HA_CREATE_UNIQUE_INDEX_BY_SORT); + MYISAM_SHARE *share=file->s; + MI_KEYDEF *key=share->keyinfo; + uint i; + /* + Deactivate all indexes that can be recreated fast. + These include packed keys on which sorting will use more temporary + space than the max allowed file length or for which the unpacked keys + will take much more space than packed keys. + Note that 'rows' may be zero for the case when we don't know how many + rows we will put into the file. + Long Unique Index (HA_KEY_ALG_LONG_HASH) will not be disabled because + there unique property is enforced at the time of ha_write_row + (check_duplicate_long_entries). So we need active index at the time of + insert. + */ + DBUG_ASSERT(file->state->records == 0 && + (!rows || rows >= MI_MIN_ROWS_TO_DISABLE_INDEXES)); + for (i=0 ; i < share->base.keys ; i++,key++) + { + if (!(key->flag & (HA_SPATIAL | HA_AUTO_KEY)) && + ! mi_too_big_key_for_sort(key,rows) && file->s->base.auto_key != i+1 && + (all_keys || !(key->flag & HA_NOSAME)) && + table->key_info[i].algorithm != HA_KEY_ALG_LONG_HASH) + { + mi_clear_key_active(share->state.key_map, i); + index_disabled= 1; + file->update|= HA_STATE_CHANGED; + file->create_unique_index_by_sort= all_keys; + } + } + } + } + else + { + if (!file->bulk_insert && + (!rows || rows >= MI_MIN_ROWS_TO_USE_BULK_INSERT)) + { + mi_init_bulk_insert(file, (size_t) thd->variables.bulk_insert_buff_size, + rows); + } + } + can_enable_indexes= index_disabled; + DBUG_VOID_RETURN; +} + +/* + end special bulk-insert optimizations, + which have been activated by start_bulk_insert(). + + SYNOPSIS + end_bulk_insert(fatal_error) + abort 0 normal end, store everything + 1 abort quickly. No need to flush/write anything. Table will be deleted + + RETURN + 0 OK + != 0 Error +*/ + +int ha_myisam::end_bulk_insert() +{ + int first_error, error; + my_bool abort= file->s->deleting; + DBUG_ENTER("ha_myisam::end_bulk_insert"); + + if ((first_error= mi_end_bulk_insert(file, abort))) + abort= 1; + + if ((error= mi_extra(file, HA_EXTRA_NO_CACHE, 0))) + { + first_error= first_error ? first_error : error; + abort= 1; + } + + if (!abort) + { + if (can_enable_indexes) + { + /* + Truncate the table when enable index operation is killed. + After truncating the table we don't need to enable the + indexes, because the last repair operation is aborted after + setting the indexes as active and trying to recreate them. + */ + + if (((first_error= enable_indexes(HA_KEY_SWITCH_NONUNIQ_SAVE)) != 0) && + table->in_use->killed) + { + delete_all_rows(); + /* not crashed, despite being killed during repair */ + file->s->state.changed&= ~(STATE_CRASHED|STATE_CRASHED_ON_REPAIR); + } + } + can_enable_indexes= 0; + } + DBUG_PRINT("exit", ("first_error: %d", first_error)); + DBUG_RETURN(first_error); +} + + +bool ha_myisam::check_and_repair(THD *thd) +{ + int error=0; + int marked_crashed; + HA_CHECK_OPT check_opt; + DBUG_ENTER("ha_myisam::check_and_repair"); + + check_opt.init(); + check_opt.flags= T_MEDIUM | T_AUTO_REPAIR; + // Don't use quick if deleted rows + if (!file->state->del && (myisam_recover_options & HA_RECOVER_QUICK)) + check_opt.flags|=T_QUICK; + sql_print_warning("Checking table: '%s'",table->s->path.str); + + const CSET_STRING query_backup= thd->query_string; + thd->set_query((char*) table->s->table_name.str, + (uint) table->s->table_name.length, system_charset_info); + + if ((marked_crashed= mi_is_crashed(file)) || check(thd, &check_opt)) + { + bool save_log_all_errors; + sql_print_warning("Recovering table: '%s'",table->s->path.str); + save_log_all_errors= thd->log_all_errors; + thd->log_all_errors|= (thd->variables.log_warnings > 2); + if (myisam_recover_options & HA_RECOVER_FULL_BACKUP) + { + char buff[MY_BACKUP_NAME_EXTRA_LENGTH+1]; + my_create_backup_name(buff, "", check_opt.start_time); + sql_print_information("Making backup of index file %s with extension '%s'", + file->s->index_file_name, buff); + mi_make_backup_of_index(file, check_opt.start_time, + MYF(MY_WME | ME_WARNING)); + } + check_opt.flags= + (((myisam_recover_options & + (HA_RECOVER_BACKUP | HA_RECOVER_FULL_BACKUP)) ? T_BACKUP_DATA : 0) | + (marked_crashed ? 0 : T_QUICK) | + (myisam_recover_options & HA_RECOVER_FORCE ? 0 : T_SAFE_REPAIR) | + T_AUTO_REPAIR); + if (repair(thd, &check_opt)) + error=1; + thd->log_all_errors= save_log_all_errors; + } + thd->set_query(query_backup); + DBUG_RETURN(error); +} + +bool ha_myisam::is_crashed() const +{ + return (file->s->state.changed & STATE_CRASHED || + (my_disable_locking && file->s->state.open_count)); +} + +int ha_myisam::update_row(const uchar *old_data, const uchar *new_data) +{ + return mi_update(file,old_data,new_data); +} + +int ha_myisam::delete_row(const uchar *buf) +{ + return mi_delete(file,buf); +} + + +int ha_myisam::index_init(uint idx, bool sorted) +{ + active_index=idx; + if (pushed_idx_cond_keyno == idx) + mi_set_index_cond_func(file, handler_index_cond_check, this); + if (pushed_rowid_filter) + mi_set_rowid_filter_func(file, handler_rowid_filter_check, + handler_rowid_filter_is_active, this); + return 0; +} + + +int ha_myisam::index_end() +{ + DBUG_ENTER("ha_myisam::index_end"); + active_index=MAX_KEY; + //pushed_idx_cond_keyno= MAX_KEY; + mi_set_index_cond_func(file, NULL, 0); + in_range_check_pushed_down= FALSE; + mi_set_rowid_filter_func(file, NULL, NULL, 0); + ds_mrr.dsmrr_close(); +#if !defined(DBUG_OFF) && defined(SQL_SELECT_FIXED_FOR_UPDATE) + file->update&= ~HA_STATE_AKTIV; // Forget active row +#endif + DBUG_RETURN(0); +} + +int ha_myisam::rnd_end() +{ + DBUG_ENTER("ha_myisam::rnd_end"); + ds_mrr.dsmrr_close(); +#if !defined(DBUG_OFF) && defined(SQL_SELECT_FIXED_FOR_UPDATE) + file->update&= ~HA_STATE_AKTIV; // Forget active row +#endif + DBUG_RETURN(0); +} + +int ha_myisam::index_read_map(uchar *buf, const uchar *key, + key_part_map keypart_map, + enum ha_rkey_function find_flag) +{ + DBUG_ASSERT(inited==INDEX); + int error=mi_rkey(file, buf, active_index, key, keypart_map, find_flag); + return error; +} + +int ha_myisam::index_read_idx_map(uchar *buf, uint index, const uchar *key, + key_part_map keypart_map, + enum ha_rkey_function find_flag) +{ + int res; + /* Use the pushed index condition if it matches the index we're scanning */ + end_range= NULL; + if (index == pushed_idx_cond_keyno) + mi_set_index_cond_func(file, handler_index_cond_check, this); + if (pushed_rowid_filter) + mi_set_rowid_filter_func(file, handler_rowid_filter_check, + handler_rowid_filter_is_active, this); + res= mi_rkey(file, buf, index, key, keypart_map, find_flag); + mi_set_index_cond_func(file, NULL, 0); + return res; +} + +int ha_myisam::index_next(uchar *buf) +{ + DBUG_ASSERT(inited==INDEX); + int error=mi_rnext(file,buf,active_index); + return error; +} + +int ha_myisam::index_prev(uchar *buf) +{ + DBUG_ASSERT(inited==INDEX); + int error=mi_rprev(file,buf, active_index); + return error; +} + +int ha_myisam::index_first(uchar *buf) +{ + DBUG_ASSERT(inited==INDEX); + int error=mi_rfirst(file, buf, active_index); + return error; +} + +int ha_myisam::index_last(uchar *buf) +{ + DBUG_ASSERT(inited==INDEX); + int error=mi_rlast(file, buf, active_index); + return error; +} + +int ha_myisam::index_next_same(uchar *buf, + const uchar *key __attribute__((unused)), + uint length __attribute__((unused))) +{ + int error; + DBUG_ASSERT(inited==INDEX); + do + { + error= mi_rnext_same(file,buf); + } while (error == HA_ERR_RECORD_DELETED); + return error; +} + + +int ha_myisam::rnd_init(bool scan) +{ + if (scan) + return mi_scan_init(file); + return mi_reset(file); // Free buffers +} + +int ha_myisam::rnd_next(uchar *buf) +{ + int error=mi_scan(file, buf); + return error; +} + +int ha_myisam::remember_rnd_pos() +{ + position((uchar*) 0); + return 0; +} + +int ha_myisam::restart_rnd_next(uchar *buf) +{ + return rnd_pos(buf, ref); +} + +int ha_myisam::rnd_pos(uchar *buf, uchar *pos) +{ + int error=mi_rrnd(file, buf, my_get_ptr(pos,ref_length)); + return error; +} + +void ha_myisam::position(const uchar *record) +{ + my_off_t row_position= mi_position(file); + my_store_ptr(ref, ref_length, row_position); + file->update|= HA_STATE_AKTIV; // Row can be updated +} + +int ha_myisam::info(uint flag) +{ + MI_ISAMINFO misam_info; + + if (!table) + return 1; + + (void) mi_status(file,&misam_info,flag); + if (flag & HA_STATUS_VARIABLE) + { + stats.records= misam_info.records; + stats.deleted= misam_info.deleted; + stats.data_file_length= misam_info.data_file_length; + stats.index_file_length= misam_info.index_file_length; + stats.delete_length= misam_info.delete_length; + stats.check_time= (ulong) misam_info.check_time; + stats.mean_rec_length= misam_info.mean_reclength; + stats.checksum= file->state->checksum; + } + if (flag & HA_STATUS_CONST) + { + TABLE_SHARE *share= table->s; + stats.max_data_file_length= misam_info.max_data_file_length; + stats.max_index_file_length= misam_info.max_index_file_length; + stats.create_time= (ulong) misam_info.create_time; + /* + We want the value of stats.mrr_length_per_rec to be platform independent. + The size of the chunk at the end of the join buffer used for MRR needs + is calculated now basing on the values passed in the stats structure. + The remaining part of the join buffer is used for records. A different + number of records in the buffer results in a different number of buffer + refills and in a different order of records in the result set. + */ + stats.mrr_length_per_rec= misam_info.reflength + 8; // 8=MY_MAX(sizeof(void *)) + + ref_length= misam_info.reflength; + share->db_options_in_use= misam_info.options; + /* record block size. We adjust with IO_SIZE to not make it too small */ + stats.block_size= MY_MAX(myisam_block_size, IO_SIZE); + + if (table_share->tmp_table == NO_TMP_TABLE) + mysql_mutex_lock(&table_share->LOCK_share); + share->keys_in_use.set_prefix(share->keys); + share->keys_in_use.intersect_extended(misam_info.key_map); + share->keys_for_keyread.intersect(share->keys_in_use); + share->db_record_offset= misam_info.record_offset; + if (share->key_parts) + { + ulong *from= misam_info.rec_per_key; + KEY *key, *key_end; + for (key= table->key_info, key_end= key + share->keys; + key < key_end ; key++) + { + memcpy(key->rec_per_key, from, + key->user_defined_key_parts * sizeof(*from)); + from+= key->user_defined_key_parts; + } + } + if (table_share->tmp_table == NO_TMP_TABLE) + mysql_mutex_unlock(&table_share->LOCK_share); + } + if (flag & HA_STATUS_ERRKEY) + { + errkey = misam_info.errkey; + my_store_ptr(dup_ref, ref_length, misam_info.dupp_key_pos); + } + if (flag & HA_STATUS_TIME) + stats.update_time = (ulong) misam_info.update_time; + if (flag & HA_STATUS_AUTO) + stats.auto_increment_value= misam_info.auto_increment; + + return 0; +} + + +int ha_myisam::extra(enum ha_extra_function operation) +{ + if ((operation == HA_EXTRA_MMAP && !opt_myisam_use_mmap) || + (operation == HA_EXTRA_WRITE_CACHE && has_long_unique())) + return 0; + return mi_extra(file, operation, 0); +} + + +int ha_myisam::reset(void) +{ + mi_set_index_cond_func(file, NULL, 0); + ds_mrr.dsmrr_close(); + return mi_reset(file); +} + +/* To be used with WRITE_CACHE and EXTRA_CACHE */ + +int ha_myisam::extra_opt(enum ha_extra_function operation, ulong cache_size) +{ + return mi_extra(file, operation, (void*) &cache_size); +} + +int ha_myisam::delete_all_rows() +{ + return mi_delete_all_rows(file); +} + + +int ha_myisam::reset_auto_increment(ulonglong value) +{ + file->s->state.auto_increment= value; + return 0; +} + +int ha_myisam::delete_table(const char *name) +{ + return mi_delete_table(name); +} + +void ha_myisam::change_table_ptr(TABLE *table_arg, TABLE_SHARE *share) +{ + handler::change_table_ptr(table_arg, share); + if (file) + file->external_ref= table_arg; +} + + +int ha_myisam::external_lock(THD *thd, int lock_type) +{ + file->in_use.data= thd; + file->external_ref= (void*) table; // For mi_killed() + return mi_lock_database(file, !table->s->tmp_table ? + lock_type : ((lock_type == F_UNLCK) ? + F_UNLCK : F_EXTRA_LCK)); +} + +THR_LOCK_DATA **ha_myisam::store_lock(THD *thd, + THR_LOCK_DATA **to, + enum thr_lock_type lock_type) +{ + if (lock_type != TL_IGNORE && file->lock.type == TL_UNLOCK) + file->lock.type=lock_type; + *to++= &file->lock; + return to; +} + +void ha_myisam::update_create_info(HA_CREATE_INFO *create_info) +{ + ha_myisam::info(HA_STATUS_AUTO | HA_STATUS_CONST); + if (!(create_info->used_fields & HA_CREATE_USED_AUTO)) + { + create_info->auto_increment_value= stats.auto_increment_value; + } + create_info->data_file_name=data_file_name; + create_info->index_file_name=index_file_name; +} + + +int ha_myisam::create(const char *name, TABLE *table_arg, + HA_CREATE_INFO *ha_create_info) +{ + int error; + uint create_flags= 0, record_count, i; + char buff[FN_REFLEN]; + MI_KEYDEF *keydef; + MI_COLUMNDEF *recinfo; + MI_CREATE_INFO create_info; + TABLE_SHARE *share= table_arg->s; + uint options= share->db_options_in_use; + DBUG_ENTER("ha_myisam::create"); + + for (i= 0; i < share->virtual_fields && !create_flags; i++) + if (table_arg->vfield[i]->flags & PART_KEY_FLAG) + create_flags|= HA_CREATE_RELIES_ON_SQL_LAYER; + for (i= 0; i < share->keys && !create_flags; i++) + if (table_arg->key_info[i].flags & HA_USES_PARSER) + create_flags|= HA_CREATE_RELIES_ON_SQL_LAYER; + + if ((error= table2myisam(table_arg, &keydef, &recinfo, &record_count))) + DBUG_RETURN(error); /* purecov: inspected */ + +#ifndef DBUG_OFF + DBUG_EXECUTE_IF("key", + Debug_key_myisam::print_keys_myisam(table_arg->in_use, + "ha_myisam::create: ", + table_arg, keydef, share->keys); + ); +#endif + + bzero((char*) &create_info, sizeof(create_info)); + create_info.max_rows= share->max_rows; + create_info.reloc_rows= share->min_rows; + create_info.with_auto_increment= share->next_number_key_offset == 0; + create_info.auto_increment= (ha_create_info->auto_increment_value ? + ha_create_info->auto_increment_value -1 : + (ulonglong) 0); + create_info.data_file_length= ((ulonglong) share->max_rows * + share->avg_row_length); + create_info.language= share->table_charset->number; + +#ifdef HAVE_READLINK + if (my_use_symdir) + { + create_info.data_file_name= ha_create_info->data_file_name; + create_info.index_file_name= ha_create_info->index_file_name; + } + else +#endif /* HAVE_READLINK */ + { + THD *thd= table_arg->in_use; + if (ha_create_info->data_file_name) + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + WARN_OPTION_IGNORED, + ER_THD(thd, WARN_OPTION_IGNORED), + "DATA DIRECTORY"); + if (ha_create_info->index_file_name) + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + WARN_OPTION_IGNORED, + ER_THD(thd, WARN_OPTION_IGNORED), + "INDEX DIRECTORY"); + } + + if (ha_create_info->tmp_table()) + create_flags|= HA_CREATE_TMP_TABLE | HA_CREATE_DELAY_KEY_WRITE; + if (ha_create_info->options & HA_CREATE_KEEP_FILES) + create_flags|= HA_CREATE_KEEP_FILES; + if (options & HA_OPTION_PACK_RECORD) + create_flags|= HA_PACK_RECORD; + if (options & HA_OPTION_CHECKSUM) + create_flags|= HA_CREATE_CHECKSUM; + if (options & HA_OPTION_DELAY_KEY_WRITE) + create_flags|= HA_CREATE_DELAY_KEY_WRITE; + + /* TODO: Check that the following fn_format is really needed */ + error= mi_create(fn_format(buff, name, "", "", + MY_UNPACK_FILENAME|MY_APPEND_EXT), + share->keys, keydef, + record_count, recinfo, + 0, (MI_UNIQUEDEF*) 0, + &create_info, create_flags); + my_free(recinfo); + DBUG_RETURN(error); +} + + +int ha_myisam::rename_table(const char * from, const char * to) +{ + return mi_rename(from,to); +} + + +void ha_myisam::get_auto_increment(ulonglong offset, ulonglong increment, + ulonglong nb_desired_values, + ulonglong *first_value, + ulonglong *nb_reserved_values) +{ + ulonglong nr; + int error; + uchar key[HA_MAX_KEY_LENGTH]; + enum ha_rkey_function search_flag= HA_READ_PREFIX_LAST; + + if (!table->s->next_number_key_offset) + { // Autoincrement at key-start + ha_myisam::info(HA_STATUS_AUTO); + *first_value= stats.auto_increment_value; + /* MyISAM has only table-level lock, so reserves to +inf */ + *nb_reserved_values= ULONGLONG_MAX; + return; + } + + /* it's safe to call the following if bulk_insert isn't on */ + mi_flush_bulk_insert(file, table->s->next_number_index); + + if (unlikely(table->key_info[table->s->next_number_index]. + key_part[table->s->next_number_keypart].key_part_flag & + HA_REVERSE_SORT)) + search_flag= HA_READ_KEY_EXACT; + + (void) extra(HA_EXTRA_KEYREAD); + key_copy(key, table->record[0], + table->key_info + table->s->next_number_index, + table->s->next_number_key_offset); + error= mi_rkey(file, table->record[1], (int) table->s->next_number_index, + key, make_prev_keypart_map(table->s->next_number_keypart), + search_flag); + if (error) + nr= 1; + else + { + /* Get data from record[1] */ + nr= ((ulonglong) table->next_number_field-> + val_int_offset(table->s->rec_buff_length)+1); + } + extra(HA_EXTRA_NO_KEYREAD); + *first_value= nr; + /* + MySQL needs to call us for next row: assume we are inserting ("a",null) + here, we return 3, and next this statement will want to insert ("b",null): + there is no reason why ("b",3+1) would be the good row to insert: maybe it + already exists, maybe 3+1 is too large... + */ + *nb_reserved_values= 1; +} + + +/* + Find out how many rows there is in the given range + + SYNOPSIS + records_in_range() + inx Index to use + min_key Start of range. Null pointer if from first key + max_key End of range. Null pointer if to last key + pages Store first and last page for the range in case of + b-trees. In other cases it's not touched. + + NOTES + min_key.flag can have one of the following values: + HA_READ_KEY_EXACT Include the key in the range + HA_READ_AFTER_KEY Don't include key in range + + max_key.flag can have one of the following values: + HA_READ_BEFORE_KEY Don't include key in range + HA_READ_AFTER_KEY Include all 'end_key' values in the range + + RETURN + HA_POS_ERROR Something is wrong with the index tree. + 0 There is no matching keys in the given range + number > 0 There is approximately 'number' matching rows in + the range. +*/ + +ha_rows ha_myisam::records_in_range(uint inx, const key_range *min_key, + const key_range *max_key, + page_range *pages) +{ + return (ha_rows) mi_records_in_range(file, (int) inx, min_key, max_key, + pages); +} + + +int ha_myisam::ft_read(uchar *buf) +{ + int error; + + if (!ft_handler) + return -1; + + thread_safe_increment(table->in_use->status_var.ha_read_next_count, + &LOCK_status); // why ? + + error=ft_handler->please->read_next(ft_handler,(char*) buf); + return error; +} + +enum_alter_inplace_result +ha_myisam::check_if_supported_inplace_alter(TABLE *new_table, + Alter_inplace_info *alter_info) +{ + DBUG_ENTER("ha_myisam::check_if_supported_inplace_alter"); + + const alter_table_operations readd_index= + ALTER_ADD_NON_UNIQUE_NON_PRIM_INDEX | + ALTER_DROP_NON_UNIQUE_NON_PRIM_INDEX; + const alter_table_operations readd_unique= + ALTER_ADD_UNIQUE_INDEX | + ALTER_DROP_UNIQUE_INDEX; + const alter_table_operations readd_pk= + ALTER_ADD_PK_INDEX | + ALTER_DROP_PK_INDEX; + + const alter_table_operations op= alter_info->handler_flags; + + if (op & ALTER_COLUMN_VCOL) + DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); + + /* + ha_myisam::open() updates table->key_info->block_size to be the actual + MYI index block size, overwriting user-specified value (if any). + So, the server can not reliably detect whether ALTER TABLE changes + key_block_size or not, it might think the block size was changed, + when it wasn't, and in this case the server will recreate (drop+add) + the index unnecessary. Fix it. + */ + + if (table->s->keys == new_table->s->keys && + ((op & readd_pk) == readd_pk || + (op & readd_unique) == readd_unique || + (op & readd_index) == readd_index)) + { + for (uint i=0; i < table->s->keys; i++) + { + KEY *old_key= table->key_info + i; + KEY *new_key= new_table->key_info + i; + + if (old_key->block_size == new_key->block_size) + DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); // must differ somewhere else + + if (new_key->block_size && new_key->block_size != old_key->block_size) + DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); // really changed + + /* any difference besides the block_size, and we give up */ + if (old_key->key_length != new_key->key_length || + old_key->flags != new_key->flags || + old_key->user_defined_key_parts != new_key->user_defined_key_parts || + old_key->algorithm != new_key->algorithm || + strcmp(old_key->name.str, new_key->name.str)) + DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); + + for (uint j= 0; j < old_key->user_defined_key_parts; j++) + { + KEY_PART_INFO *old_kp= old_key->key_part + j; + KEY_PART_INFO *new_kp= new_key->key_part + j; + if (old_kp->offset != new_kp->offset || + old_kp->null_offset != new_kp->null_offset || + old_kp->length != new_kp->length || + old_kp->fieldnr != new_kp->fieldnr || + old_kp->key_part_flag != new_kp->key_part_flag || + old_kp->type != new_kp->type || + old_kp->null_bit != new_kp->null_bit) + DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); + } + } + alter_info->handler_flags &= ~(readd_pk | readd_unique | readd_index); + } + DBUG_RETURN(handler::check_if_supported_inplace_alter(new_table, alter_info)); +} + + +static bool directories_differ(const char *d1, const char *d2) +{ + if (!d1 && !d2) + return false; + if (!d1 || !d2) + return true; + size_t l1= dirname_length(d1), l2= dirname_length(d2); + return l1 != l2 || strncmp(d1, d2, l1); +} + + +bool ha_myisam::check_if_incompatible_data(HA_CREATE_INFO *create_info, + uint table_changes) +{ + uint options= table->s->db_options_in_use; + + if ((create_info->used_fields & HA_CREATE_USED_AUTO && + create_info->auto_increment_value != stats.auto_increment_value) || + directories_differ(create_info->data_file_name, data_file_name) || + directories_differ(create_info->index_file_name, index_file_name) || + table_changes == IS_EQUAL_NO || + table_changes & IS_EQUAL_PACK_LENGTH) // Not implemented yet + return COMPATIBLE_DATA_NO; + + if ((options & (HA_OPTION_PACK_RECORD | HA_OPTION_CHECKSUM)) != + (create_info->table_options & (HA_OPTION_PACK_RECORD | HA_OPTION_CHECKSUM))) + return COMPATIBLE_DATA_NO; + return COMPATIBLE_DATA_YES; +} + + +/** + Check if a table is incompatible with the current version. + + The cases are: + - Table has checksum, varchars and are not of dynamic record type +*/ + +int ha_myisam::check_for_upgrade(HA_CHECK_OPT *check_opt) +{ + if ((file->s->options & HA_OPTION_CHECKSUM) && + !(file->s->options & HA_OPTION_NULL_FIELDS) && + !(file->s->options & HA_OPTION_PACK_RECORD) && + file->s->has_varchar_fields) + { + /* We need alter there to get the HA_OPTION_NULL_FIELDS flag to be set */ + return HA_ADMIN_NEEDS_ALTER; + } + return HA_ADMIN_OK; +} + + +extern int mi_panic(enum ha_panic_function flag); +int myisam_panic(handlerton *hton, ha_panic_function flag) +{ + return mi_panic(flag); +} + +static int myisam_drop_table(handlerton *hton, const char *path) +{ + return mi_delete_table(path); +} + +static int myisam_init(void *p) +{ + handlerton *hton; + + init_myisam_psi_keys(); + + /* Set global variables based on startup options */ + if (myisam_recover_options && myisam_recover_options != HA_RECOVER_OFF) + ha_open_options|=HA_OPEN_ABORT_IF_CRASHED; + else + myisam_recover_options= HA_RECOVER_OFF; + + myisam_block_size=(uint) 1 << my_bit_log2_uint64(opt_myisam_block_size); + + hton= (handlerton *)p; + hton->db_type= DB_TYPE_MYISAM; + hton->create= myisam_create_handler; + hton->drop_table= myisam_drop_table; + hton->panic= myisam_panic; + hton->flags= HTON_CAN_RECREATE | HTON_SUPPORT_LOG_TABLES; + hton->tablefile_extensions= ha_myisam_exts; + mi_killed= mi_killed_in_mariadb; + + return 0; +} + +static struct st_mysql_sys_var* myisam_sysvars[]= { + MYSQL_SYSVAR(block_size), + MYSQL_SYSVAR(data_pointer_size), + MYSQL_SYSVAR(max_sort_file_size), + MYSQL_SYSVAR(recover_options), + MYSQL_SYSVAR(repair_threads), + MYSQL_SYSVAR(sort_buffer_size), + MYSQL_SYSVAR(use_mmap), + MYSQL_SYSVAR(mmap_size), + MYSQL_SYSVAR(stats_method), + 0 +}; + +/**************************************************************************** + * MyISAM MRR implementation: use DS-MRR + ***************************************************************************/ + +int ha_myisam::multi_range_read_init(RANGE_SEQ_IF *seq, void *seq_init_param, + uint n_ranges, uint mode, + HANDLER_BUFFER *buf) +{ + return ds_mrr.dsmrr_init(this, seq, seq_init_param, n_ranges, mode, buf); +} + +int ha_myisam::multi_range_read_next(range_id_t *range_info) +{ + return ds_mrr.dsmrr_next(range_info); +} + +ha_rows ha_myisam::multi_range_read_info_const(uint keyno, RANGE_SEQ_IF *seq, + void *seq_init_param, + uint n_ranges, uint *bufsz, + uint *flags, Cost_estimate *cost) +{ + /* + This call is here because there is no location where this->table would + already be known. + TODO: consider moving it into some per-query initialization call. + */ + ds_mrr.init(this, table); + return ds_mrr.dsmrr_info_const(keyno, seq, seq_init_param, n_ranges, bufsz, + flags, cost); +} + +ha_rows ha_myisam::multi_range_read_info(uint keyno, uint n_ranges, uint keys, + uint key_parts, uint *bufsz, + uint *flags, Cost_estimate *cost) +{ + ds_mrr.init(this, table); + return ds_mrr.dsmrr_info(keyno, n_ranges, keys, key_parts, bufsz, flags, cost); +} + + +int ha_myisam::multi_range_read_explain_info(uint mrr_mode, char *str, + size_t size) +{ + return ds_mrr.dsmrr_explain_info(mrr_mode, str, size); +} + +/* MyISAM MRR implementation ends */ + + +/* Index condition pushdown implementation*/ + + +Item *ha_myisam::idx_cond_push(uint keyno_arg, Item* idx_cond_arg) +{ + /* + Check if the key contains a blob field. If it does then MyISAM + should not accept the pushed index condition since MyISAM will not + read the blob field from the index entry during evaluation of the + pushed index condition and the BLOB field might be part of the + range evaluation done by the ICP code. + */ + const KEY *key= &table_share->key_info[keyno_arg]; + + for (uint k= 0; k < key->user_defined_key_parts; ++k) + { + const KEY_PART_INFO *key_part= &key->key_part[k]; + if (key_part->key_part_flag & HA_BLOB_PART) + { + /* Let the server handle the index condition */ + return idx_cond_arg; + } + } + + pushed_idx_cond_keyno= keyno_arg; + pushed_idx_cond= idx_cond_arg; + in_range_check_pushed_down= TRUE; + if (active_index == pushed_idx_cond_keyno) + mi_set_index_cond_func(file, handler_index_cond_check, this); + return NULL; +} + +bool ha_myisam::rowid_filter_push(Rowid_filter* rowid_filter) +{ + pushed_rowid_filter= rowid_filter; + mi_set_rowid_filter_func(file, handler_rowid_filter_check, + handler_rowid_filter_is_active, this); + return false; +} + +struct st_mysql_storage_engine myisam_storage_engine= +{ MYSQL_HANDLERTON_INTERFACE_VERSION }; + +maria_declare_plugin(myisam) +{ + MYSQL_STORAGE_ENGINE_PLUGIN, + &myisam_storage_engine, + "MyISAM", + "MySQL AB", + "Non-transactional engine with good performance and small data footprint", + PLUGIN_LICENSE_GPL, + myisam_init, /* Plugin Init */ + NULL, /* Plugin Deinit */ + 0x0100, /* 1.0 */ + NULL, /* status variables */ + myisam_sysvars, /* system variables */ + "1.0", /* string version */ + MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ +} +maria_declare_plugin_end; + + +#ifdef HAVE_QUERY_CACHE +/** + @brief Register a named table with a call back function to the query cache. + + @param thd The thread handle + @param table_key A pointer to the table name in the table cache + @param key_length The length of the table name + @param[out] engine_callback The pointer to the storage engine call back + function, currently 0 + @param[out] engine_data Engine data will be set to 0. + + @note Despite the name of this function, it is used to check each statement + before it is cached and not to register a table or callback function. + + @see handler::register_query_cache_table + + @return The error code. The engine_data and engine_callback will be set to 0. + @retval TRUE Success + @retval FALSE An error occurred +*/ + +my_bool ha_myisam::register_query_cache_table(THD *thd, const char *table_name, + uint table_name_len, + qc_engine_callback + *engine_callback, + ulonglong *engine_data) +{ + DBUG_ENTER("ha_myisam::register_query_cache_table"); + /* + No call back function is needed to determine if a cached statement + is valid or not. + */ + *engine_callback= 0; + + /* + No engine data is needed. + */ + *engine_data= 0; + + if (file->s->concurrent_insert) + { + /* + If a concurrent INSERT has happened just before the currently + processed SELECT statement, the total size of the table is + unknown. + + To determine if the table size is known, the current thread's snap + shot of the table size with the actual table size are compared. + + If the table size is unknown the SELECT statement can't be cached. + + When concurrent inserts are disabled at table open, mi_ondopen() + does not assign a get_status() function. In this case the local + ("current") status is never updated. We would wrongly think that + we cannot cache the statement. + */ + ulonglong actual_data_file_length; + ulonglong current_data_file_length; + + /* + POSIX visibility rules specify that "2. Whatever memory values a + thread can see when it unlocks a mutex <...> can also be seen by any + thread that later locks the same mutex". In this particular case, + concurrent insert thread had modified the data_file_length in + MYISAM_SHARE before it has unlocked (or even locked) + structure_guard_mutex. So, here we're guaranteed to see at least that + value after we've locked the same mutex. We can see a later value + (modified by some other thread) though, but it's ok, as we only want + to know if the variable was changed, the actual new value doesn't matter + */ + actual_data_file_length= file->s->state.state.data_file_length; + current_data_file_length= file->save_state.data_file_length; + + if (current_data_file_length != actual_data_file_length) + { + /* Don't cache current statement. */ + DBUG_RETURN(FALSE); + } + } + + /* + This query execution might have started after the query cache was flushed + by a concurrent INSERT. In this case, don't cache this statement as the + data file length difference might not be visible yet if the tables haven't + been unlocked by the concurrent insert thread. + */ + if (file->state->uncacheable) + DBUG_RETURN(FALSE); + + /* It is ok to try to cache current statement. */ + DBUG_RETURN(TRUE); +} +#endif diff --git a/storage/myisam/ha_myisam.h b/storage/myisam/ha_myisam.h new file mode 100644 index 00000000..c4c46a63 --- /dev/null +++ b/storage/myisam/ha_myisam.h @@ -0,0 +1,180 @@ +/* + Copyright (c) 2000, 2012, Oracle and/or its affiliates. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + + +#ifdef USE_PRAGMA_INTERFACE +#pragma interface /* gcc class implementation */ +#endif + +/* class for the the myisam handler */ + +#include <myisam.h> +#include <ft_global.h> +#include "handler.h" /* handler */ +#include "table.h" /* TABLE_SHARE */ + +#define HA_RECOVER_DEFAULT 1 /* Automatic recover active */ +#define HA_RECOVER_BACKUP 2 /* Make a backupfile on recover */ +#define HA_RECOVER_FORCE 4 /* Recover even if we loose rows */ +#define HA_RECOVER_QUICK 8 /* Don't check rows in data file */ +#define HA_RECOVER_FULL_BACKUP 16 /* Make a copy of index file too */ +#define HA_RECOVER_OFF 32 /* No automatic recover */ + +extern TYPELIB myisam_recover_typelib; +extern const char *myisam_recover_names[]; +extern ulonglong myisam_recover_options; + +C_MODE_START +check_result_t index_cond_func_myisam(void *arg); +C_MODE_END + +class ha_myisam final : public handler +{ + MI_INFO *file; + ulonglong int_table_flags; + char *data_file_name, *index_file_name; + bool can_enable_indexes; + int repair(THD *thd, HA_CHECK ¶m, bool optimize); + void setup_vcols_for_repair(HA_CHECK *param); + void restore_vcos_after_repair(); + + public: + ha_myisam(handlerton *hton, TABLE_SHARE *table_arg); + ~ha_myisam() = default; + handler *clone(const char *name, MEM_ROOT *mem_root); + const char *index_type(uint key_number); + ulonglong table_flags() const { return int_table_flags; } + int index_init(uint idx, bool sorted); + int index_end(); + int rnd_end(); + + ulong index_flags(uint inx, uint part, bool all_parts) const; + uint max_supported_keys() const { return MI_MAX_KEY; } + uint max_supported_key_parts() const { return HA_MAX_KEY_SEG; } + uint max_supported_key_length() const { return HA_MAX_KEY_LENGTH; } + uint max_supported_key_part_length() const { return HA_MAX_KEY_LENGTH; } + void change_table_ptr(TABLE *table_arg, TABLE_SHARE *share); + int open(const char *name, int mode, uint test_if_locked); + int close(void); + int write_row(const uchar * buf); + int update_row(const uchar * old_data, const uchar * new_data); + int delete_row(const uchar * buf); + int index_read_map(uchar *buf, const uchar *key, key_part_map keypart_map, + enum ha_rkey_function find_flag); + int index_read_idx_map(uchar *buf, uint index, const uchar *key, + key_part_map keypart_map, + enum ha_rkey_function find_flag); + int index_next(uchar * buf); + int index_prev(uchar * buf); + int index_first(uchar * buf); + int index_last(uchar * buf); + int index_next_same(uchar *buf, const uchar *key, uint keylen); + int ft_init() + { + if (!ft_handler) + return 1; + ft_handler->please->reinit_search(ft_handler); + return 0; + } + FT_INFO *ft_init_ext(uint flags, uint inx,String *key) + { + return ft_init_search(flags,file,inx, + (uchar *)key->ptr(), key->length(), key->charset(), + table->record[0]); + } + int ft_read(uchar *buf); + int rnd_init(bool scan); + int rnd_next(uchar *buf); + int rnd_pos(uchar * buf, uchar *pos); + int remember_rnd_pos(); + int restart_rnd_next(uchar *buf); + void position(const uchar *record); + int info(uint); + int extra(enum ha_extra_function operation); + int extra_opt(enum ha_extra_function operation, ulong cache_size); + int reset(void); + int external_lock(THD *thd, int lock_type); + int delete_all_rows(void); + int reset_auto_increment(ulonglong value); + int disable_indexes(uint mode); + int enable_indexes(uint mode); + int indexes_are_disabled(void); + void start_bulk_insert(ha_rows rows, uint flags); + int end_bulk_insert(); + ha_rows records_in_range(uint inx, const key_range *min_key, + const key_range *max_key, page_range *pages); + void update_create_info(HA_CREATE_INFO *create_info); + int create(const char *name, TABLE *form, HA_CREATE_INFO *create_info); + THR_LOCK_DATA **store_lock(THD *thd, THR_LOCK_DATA **to, + enum thr_lock_type lock_type); + virtual void get_auto_increment(ulonglong offset, ulonglong increment, + ulonglong nb_desired_values, + ulonglong *first_value, + ulonglong *nb_reserved_values); + int rename_table(const char * from, const char * to); + int delete_table(const char *name); + int check_for_upgrade(HA_CHECK_OPT *check_opt); + int check(THD* thd, HA_CHECK_OPT* check_opt); + int analyze(THD* thd,HA_CHECK_OPT* check_opt); + int repair(THD* thd, HA_CHECK_OPT* check_opt); + bool check_and_repair(THD *thd); + bool is_crashed() const; + bool auto_repair(int error) const + { + return (myisam_recover_options != HA_RECOVER_OFF && + error == HA_ERR_CRASHED_ON_USAGE); + } + int optimize(THD* thd, HA_CHECK_OPT* check_opt); + int assign_to_keycache(THD* thd, HA_CHECK_OPT* check_opt); + int preload_keys(THD* thd, HA_CHECK_OPT* check_opt); + enum_alter_inplace_result check_if_supported_inplace_alter(TABLE *new_table, + Alter_inplace_info *alter_info); + bool check_if_incompatible_data(HA_CREATE_INFO *info, uint table_changes); +#ifdef HAVE_QUERY_CACHE + my_bool register_query_cache_table(THD *thd, const char *table_key, + uint key_length, + qc_engine_callback + *engine_callback, + ulonglong *engine_data); +#endif + MI_INFO *file_ptr(void) + { + return file; + } +public: + /** + * Multi Range Read interface + */ + int multi_range_read_init(RANGE_SEQ_IF *seq, void *seq_init_param, + uint n_ranges, uint mode, HANDLER_BUFFER *buf); + int multi_range_read_next(range_id_t *range_info); + ha_rows multi_range_read_info_const(uint keyno, RANGE_SEQ_IF *seq, + void *seq_init_param, + uint n_ranges, uint *bufsz, + uint *flags, Cost_estimate *cost); + ha_rows multi_range_read_info(uint keyno, uint n_ranges, uint keys, + uint key_parts, uint *bufsz, + uint *flags, Cost_estimate *cost); + int multi_range_read_explain_info(uint mrr_mode, char *str, size_t size); + + /* Index condition pushdown implementation */ + Item *idx_cond_push(uint keyno, Item* idx_cond); + bool rowid_filter_push(Rowid_filter* rowid_filter); + +private: + DsMrr_impl ds_mrr; + friend check_result_t index_cond_func_myisam(void *arg); +}; diff --git a/storage/myisam/mi_cache.c b/storage/myisam/mi_cache.c new file mode 100644 index 00000000..9a6c4044 --- /dev/null +++ b/storage/myisam/mi_cache.c @@ -0,0 +1,109 @@ +/* Copyright (c) 2000-2003, 2005-2007 MySQL AB, 2009 Sun Microsystems, Inc. + Use is subject to license terms. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* + Functions for read record cacheing with myisam + Used for reading dynamic/compressed records from datafile. + + Can fetch data directly from file (outside cache), + if reading a small chunk straight before the cached part (with possible + overlap). + + Can be explicitly asked not to use cache (by not setting READING_NEXT in + flag) - useful for occasional out-of-cache reads, when the next read is + expected to hit the cache again. + + Allows "partial read" errors in the record header (when READING_HEADER flag + is set) - unread part is bzero'ed + + Note: out-of-cache reads are enabled for shared IO_CACHE's too, + as these reads will be cached by OS cache (and mysql_file_pread is always atomic) +*/ + +#include "myisamdef.h" + +int _mi_read_cache(IO_CACHE *info, uchar *buff, my_off_t pos, size_t length, + int flag) +{ + size_t read_length,in_buff_length; + my_off_t offset; + uchar *in_buff_pos; + DBUG_ENTER("_mi_read_cache"); + DBUG_ASSERT(!(info->myflags & MY_ENCRYPT)); + + if (pos < info->pos_in_file) + { + read_length=length; + if ((my_off_t) read_length > (my_off_t) (info->pos_in_file-pos)) + read_length=(size_t)(info->pos_in_file-pos); + info->seek_not_done=1; + if (mysql_file_pread(info->file, buff, read_length, pos, MYF(MY_NABP))) + DBUG_RETURN(1); + if (!(length-=read_length)) + DBUG_RETURN(0); + pos+=read_length; + buff+=read_length; + } + if (pos >= info->pos_in_file && + (offset= (my_off_t) (pos - info->pos_in_file)) < + (my_off_t) (info->read_end - info->request_pos)) + { + in_buff_pos=info->request_pos+ (uint)offset; + in_buff_length= MY_MIN(length, (size_t)(info->read_end-in_buff_pos)); + memcpy(buff,info->request_pos+(uint) offset, in_buff_length); + if (!(length-=in_buff_length)) + DBUG_RETURN(0); + pos+=in_buff_length; + buff+=in_buff_length; + } + else + in_buff_length=0; + if (flag & READING_NEXT) + { + if (pos != (info->pos_in_file + + (uint) (info->read_end - info->request_pos))) + { + info->pos_in_file=pos; /* Force start here */ + info->read_pos=info->read_end=info->request_pos; /* Everything used */ + info->seek_not_done=1; + } + else + info->read_pos=info->read_end; /* All block used */ + if (!_my_b_read(info,buff,length)) + DBUG_RETURN(0); + read_length=info->error; + } + else + { + info->seek_not_done=1; + if ((read_length= mysql_file_pread(info->file, buff, length, pos, + MYF(0))) == length) + DBUG_RETURN(0); + } + if (!(flag & READING_HEADER) || (int) read_length == -1 || + read_length+in_buff_length < 3) + { + DBUG_PRINT("error", + ("Error %d reading next-multi-part block (Got %d bytes)", + my_errno, (int) read_length)); + if (!my_errno || my_errno == -1 || my_errno == HA_ERR_FILE_TOO_SHORT) + my_errno= HA_ERR_WRONG_IN_RECORD; + DBUG_RETURN(1); + } + bzero(buff+read_length,MI_BLOCK_INFO_HEADER_LENGTH - in_buff_length - + read_length); + DBUG_RETURN(0); +} /* _mi_read_cache */ diff --git a/storage/myisam/mi_changed.c b/storage/myisam/mi_changed.c new file mode 100644 index 00000000..1eb2b517 --- /dev/null +++ b/storage/myisam/mi_changed.c @@ -0,0 +1,34 @@ +/* Copyright (c) 2000, 2001, 2005, 2006 MySQL AB, 2009 Sun Microsystems, Inc. + Use is subject to license terms. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Check if somebody has changed table since last check. */ + +#include "myisamdef.h" + + /* Return 0 if table isn't changed */ + +int mi_is_changed(MI_INFO *info) +{ + int result; + DBUG_ENTER("mi_is_changed"); + if (fast_mi_readinfo(info)) + DBUG_RETURN(-1); + (void) _mi_writeinfo(info,0); + result=(int) info->data_changed; + info->data_changed=0; + DBUG_PRINT("exit",("result: %d",result)); + DBUG_RETURN(result); +} diff --git a/storage/myisam/mi_check.c b/storage/myisam/mi_check.c new file mode 100644 index 00000000..33fab259 --- /dev/null +++ b/storage/myisam/mi_check.c @@ -0,0 +1,4797 @@ +/* Copyright (c) 2000, 2018, Oracle and/or its affiliates. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Describe, check and repair of MyISAM tables */ + +/* + About checksum calculation. + + There are two types of checksums. Table checksum and row checksum. + + Row checksum is an additional byte at the end of dynamic length + records. It must be calculated if the table is configured for them. + Otherwise they must not be used. The variable + MYISAM_SHARE::calc_checksum determines if row checksums are used. + MI_INFO::checksum is used as temporary storage during row handling. + For parallel repair we must assure that only one thread can use this + variable. There is no problem on the write side as this is done by one + thread only. But when checking a record after read this could go + wrong. But since all threads read through a common read buffer, it is + sufficient if only one thread checks it. + + Table checksum is an eight byte value in the header of the index file. + It can be calculated even if row checksums are not used. The variable + MI_CHECK::glob_crc is calculated over all records. + MI_SORT_PARAM::calc_checksum determines if this should be done. This + variable is not part of MI_CHECK because it must be set per thread for + parallel repair. The global glob_crc must be changed by one thread + only. And it is sufficient to calculate the checksum once only. +*/ + +#include "ftdefs.h" +#include <m_ctype.h> +#include <my_getopt.h> +#ifdef HAVE_SYS_VADVISE_H +#include <sys/vadvise.h> +#endif +#include "rt_index.h" +#include <mysqld_error.h> + + /* Functions defined in this file */ + +static int check_k_link(HA_CHECK *param, MI_INFO *info,uint nr); +static int chk_index(HA_CHECK *, MI_INFO *, MI_KEYDEF *, my_off_t, uchar *, + ha_rows *, ha_checksum *, uint); +static uint isam_key_length(MI_INFO *info,MI_KEYDEF *keyinfo); +static ha_checksum calc_checksum(ha_rows count); +static int writekeys(MI_SORT_PARAM *sort_param); +static int sort_one_index(HA_CHECK *, MI_INFO *, MI_KEYDEF *, my_off_t, File); +static int sort_key_read(MI_SORT_PARAM *sort_param,void *key); +static int sort_ft_key_read(MI_SORT_PARAM *sort_param,void *key); +static int sort_get_next_record(MI_SORT_PARAM *sort_param); +static int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,const void *b); +static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a); +static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a); +static my_off_t get_record_for_key(MI_INFO *, MI_KEYDEF *, uchar *); +static int sort_insert_key(MI_SORT_PARAM *, SORT_KEY_BLOCKS *, uchar *, my_off_t); +static int sort_delete_record(MI_SORT_PARAM *sort_param); +/*static int flush_pending_blocks(HA_CHECK *param);*/ +static SORT_KEY_BLOCKS *alloc_key_blocks(HA_CHECK *, uint, uint); +static ha_checksum mi_byte_checksum(const uchar *buf, uint length); +static void set_data_file_type(MI_SORT_INFO *sort_info, MYISAM_SHARE *share); +static int replace_data_file(HA_CHECK *param, MI_INFO *info, File new_file); + +void myisamchk_init(HA_CHECK *param) +{ + bzero((uchar*) param,sizeof(*param)); + /* Set all params that are not 0 */ + param->opt_follow_links=1; + param->keys_in_use= ~(ulonglong) 0; + param->search_after_block=HA_OFFSET_ERROR; + param->use_buffers= KEY_BUFFER_INIT; + param->read_buffer_length=READ_BUFFER_INIT; + param->write_buffer_length=READ_BUFFER_INIT; + param->sort_buffer_length=SORT_BUFFER_INIT; + param->sort_key_blocks=BUFFERS_WHEN_SORTING; + param->tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL; + param->myf_rw=MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL); + param->max_record_length= LONGLONG_MAX; + param->key_cache_block_size= KEY_CACHE_BLOCK_SIZE; + param->stats_method= MI_STATS_METHOD_NULLS_NOT_EQUAL; + param->need_print_msg_lock= 0; +} + + /* Check the status flags for the table */ + +int chk_status(HA_CHECK *param, register MI_INFO *info) +{ + MYISAM_SHARE *share=info->s; + + /* Protection for HA_EXTRA_FLUSH */ + mysql_mutex_lock(&share->intern_lock); + + if (mi_is_crashed_on_repair(info)) + mi_check_print_warning(param, + "Table is marked as crashed and last repair failed"); + else if (mi_is_crashed(info)) + mi_check_print_warning(param, + "Table is marked as crashed"); + if (share->state.open_count != (uint) (info->s->global_changed ? 1 : 0)) + { + /* Don't count this as a real warning, as check can correct this ! */ + my_bool save=param->warning_printed; + mi_check_print_warning(param, + share->state.open_count==1 ? + "%d client is using or hasn't closed the table properly" : + "%d clients are using or haven't closed the table properly", + share->state.open_count); + /* If this will be fixed by the check, forget the warning */ + if (param->testflag & T_UPDATE_STATE) + param->warning_printed=save; + } + mysql_mutex_unlock(&share->intern_lock); + return 0; +} + + /* Check delete links */ + +int chk_del(HA_CHECK *param, register MI_INFO *info, ulonglong test_flag) +{ + reg2 ha_rows i; + uint delete_link_length; + my_off_t empty,next_link,UNINIT_VAR(old_link); + char buff[22],buff2[22]; + DBUG_ENTER("chk_del"); + + param->record_checksum=0; + delete_link_length=((info->s->options & HA_OPTION_PACK_RECORD) ? 20 : + info->s->rec_reflength+1); + + if (!(test_flag & T_SILENT)) + puts("- check record delete-chain"); + + next_link=info->s->state.dellink; + if (info->state->del == 0) + { + if (test_flag & T_VERBOSE) + { + puts("No recordlinks"); + } + } + else + { + if (test_flag & T_VERBOSE) + printf("Recordlinks: "); + empty=0; + for (i= info->state->del ; i > 0L && next_link != HA_OFFSET_ERROR ; i--) + { + if (killed_ptr(param)) + DBUG_RETURN(1); + if (test_flag & T_VERBOSE) + printf(" %9s",llstr(next_link,buff)); + if (next_link >= info->state->data_file_length) + goto wrong; + if (mysql_file_pread(info->dfile, (uchar*) buff, delete_link_length, + next_link, MYF(MY_NABP))) + { + if (test_flag & T_VERBOSE) puts(""); + mi_check_print_error(param,"Can't read delete-link at filepos: %s", + llstr(next_link,buff)); + DBUG_RETURN(1); + } + if (*buff != '\0') + { + if (test_flag & T_VERBOSE) puts(""); + mi_check_print_error(param,"Record at pos: %s is not remove-marked", + llstr(next_link,buff)); + goto wrong; + } + if (info->s->options & HA_OPTION_PACK_RECORD) + { + my_off_t prev_link=mi_sizekorr(buff+12); + if (empty && prev_link != old_link) + { + if (test_flag & T_VERBOSE) puts(""); + mi_check_print_error(param,"Deleted block at %s doesn't point back at previous delete link",llstr(next_link,buff2)); + goto wrong; + } + old_link=next_link; + next_link=mi_sizekorr(buff+4); + empty+=mi_uint3korr(buff+1); + } + else + { + param->record_checksum+=(ha_checksum) next_link; + next_link=_mi_rec_pos(info->s,(uchar*) buff+1); + empty+=info->s->base.pack_reclength; + } + } + if (test_flag & T_VERBOSE) + puts("\n"); + if (empty != info->state->empty) + { + mi_check_print_warning(param, + "Found %s deleted space in delete link chain. Should be %s", + llstr(empty,buff2), + llstr(info->state->empty,buff)); + } + if (next_link != HA_OFFSET_ERROR) + { + mi_check_print_error(param, + "Found more than the expected %s deleted rows in delete link chain", + llstr(info->state->del, buff)); + goto wrong; + } + if (i != 0) + { + mi_check_print_error(param, + "Found %s deleted rows in delete link chain. Should be %s", + llstr(info->state->del - i, buff2), + llstr(info->state->del, buff)); + goto wrong; + } + } + DBUG_RETURN(0); + +wrong: + param->testflag|=T_RETRY_WITHOUT_QUICK; + if (test_flag & T_VERBOSE) puts(""); + mi_check_print_error(param,"record delete-link-chain corrupted"); + DBUG_RETURN(1); +} /* chk_del */ + + + /* Check delete links in index file */ + +static int check_k_link(HA_CHECK *param, register MI_INFO *info, uint nr) +{ + my_off_t next_link; + uint block_size=(nr+1)*MI_MIN_KEY_BLOCK_LENGTH; + ha_rows records; + char llbuff[21], llbuff2[21]; + uchar *buff; + DBUG_ENTER("check_k_link"); + DBUG_PRINT("enter", ("block_size: %u", block_size)); + + if (param->testflag & T_VERBOSE) + printf("block_size %4u:", block_size); /* purecov: tested */ + + next_link=info->s->state.key_del[nr]; + records= (ha_rows) (info->state->key_file_length / block_size); + while (next_link != HA_OFFSET_ERROR && records > 0) + { + if (killed_ptr(param)) + DBUG_RETURN(1); + if (param->testflag & T_VERBOSE) + printf("%16s",llstr(next_link,llbuff)); + + /* Key blocks must lay within the key file length entirely. */ + if (next_link + block_size > info->state->key_file_length) + { + /* purecov: begin tested */ + mi_check_print_error(param, "Invalid key block position: %s " + "key block size: %u file_length: %s", + llstr(next_link, llbuff), block_size, + llstr(info->state->key_file_length, llbuff2)); + DBUG_RETURN(1); + /* purecov: end */ + } + + /* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */ + if (next_link & (MI_MIN_KEY_BLOCK_LENGTH - 1)) + { + /* purecov: begin tested */ + mi_check_print_error(param, "Mis-aligned key block: %s " + "minimum key block length: %u", + llstr(next_link, llbuff), MI_MIN_KEY_BLOCK_LENGTH); + DBUG_RETURN(1); + /* purecov: end */ + } + + /* + Read the key block with MI_MIN_KEY_BLOCK_LENGTH to find next link. + If the key cache block size is smaller than block_size, we can so + avoid unnecessary eviction of cache block. + */ + if (!(buff=key_cache_read(info->s->key_cache, + info->s->kfile, next_link, DFLT_INIT_HITS, + (uchar*) info->buff, MI_MIN_KEY_BLOCK_LENGTH, + MI_MIN_KEY_BLOCK_LENGTH, 1))) + { + /* purecov: begin tested */ + mi_check_print_error(param, "key cache read error for block: %s", + llstr(next_link,llbuff)); + DBUG_RETURN(1); + /* purecov: end */ + } + next_link=mi_sizekorr(buff); + records--; + param->key_file_blocks+=block_size; + } + if (param->testflag & T_VERBOSE) + { + if (next_link != HA_OFFSET_ERROR) + printf("%16s\n",llstr(next_link,llbuff)); + else + puts(""); + } + DBUG_RETURN (next_link != HA_OFFSET_ERROR); +} /* check_k_link */ + + + /* Check sizes of files */ + +int chk_size(HA_CHECK *param, register MI_INFO *info) +{ + int error=0; + register my_off_t skr,size; + char buff[22],buff2[22]; + DBUG_ENTER("chk_size"); + + if (!(param->testflag & T_SILENT)) puts("- check file-size"); + + /* The following is needed if called externally (not from myisamchk) */ + flush_key_blocks(info->s->key_cache, + info->s->kfile, &info->s->dirty_part_map, + FLUSH_FORCE_WRITE); + + size= mysql_file_seek(info->s->kfile, 0L, MY_SEEK_END, MYF(MY_THREADSAFE)); + if ((skr=(my_off_t) info->state->key_file_length) != size) + { + /* Don't give error if file generated by myisampack */ + if (skr > size && mi_is_any_key_active(info->s->state.key_map)) + { + error=1; + mi_check_print_error(param, + "Size of indexfile is: %-8s Should be: %s", + llstr(size,buff), llstr(skr,buff2)); + } + else + mi_check_print_warning(param, + "Size of indexfile is: %-8s Should be: %s", + llstr(size,buff), llstr(skr,buff2)); + } + if (!(param->testflag & T_VERY_SILENT) && + ! (info->s->options & HA_OPTION_COMPRESS_RECORD) && + ulonglong2double(info->state->key_file_length) > + ulonglong2double(info->s->base.margin_key_file_length)*0.9) + mi_check_print_warning(param,"Keyfile is almost full, %10s of %10s used", + llstr(info->state->key_file_length,buff), + llstr(info->s->base.max_key_file_length-1,buff)); + + size= mysql_file_seek(info->dfile, 0L, MY_SEEK_END, MYF(0)); + skr=(my_off_t) info->state->data_file_length; + if (info->s->options & HA_OPTION_COMPRESS_RECORD) + skr+= MEMMAP_EXTRA_MARGIN; +#ifdef USE_RELOC + if (info->data_file_type == STATIC_RECORD && + skr < (my_off_t) info->s->base.reloc*info->s->base.min_pack_length) + skr=(my_off_t) info->s->base.reloc*info->s->base.min_pack_length; +#endif + if (skr != size) + { + info->state->data_file_length=size; /* Skip other errors */ + if (skr > size && skr != size + MEMMAP_EXTRA_MARGIN) + { + error=1; + mi_check_print_error(param,"Size of datafile is: %-9s Should be: %s", + llstr(size,buff), llstr(skr,buff2)); + param->testflag|=T_RETRY_WITHOUT_QUICK; + } + else + { + mi_check_print_warning(param, + "Size of datafile is: %-9s Should be: %s", + llstr(size,buff), llstr(skr,buff2)); + } + } + if (!(param->testflag & T_VERY_SILENT) && + !(info->s->options & HA_OPTION_COMPRESS_RECORD) && + ulonglong2double(info->state->data_file_length) > + (ulonglong2double(info->s->base.max_data_file_length)*0.9)) + mi_check_print_warning(param, "Datafile is almost full, %10s of %10s used", + llstr(info->state->data_file_length,buff), + llstr(info->s->base.max_data_file_length-1,buff2)); + DBUG_RETURN(error); +} /* chk_size */ + + + /* Check keys */ + +int chk_key(HA_CHECK *param, register MI_INFO *info) +{ + uint key,found_keys=0,full_text_keys=0,result=0; + ha_rows keys; + ha_checksum old_record_checksum,init_checksum; + my_off_t all_keydata,all_totaldata,key_totlength,length; + ulong *rec_per_key_part; + MYISAM_SHARE *share=info->s; + MI_KEYDEF *keyinfo; + char buff[22],buff2[22]; + DBUG_ENTER("chk_key"); + + if (!(param->testflag & T_SILENT)) + puts("- check key delete-chain"); + + param->key_file_blocks=info->s->base.keystart; + for (key=0 ; key < info->s->state.header.max_block_size_index ; key++) + if (check_k_link(param,info,key)) + { + if (param->testflag & T_VERBOSE) puts(""); + mi_check_print_error(param,"key delete-link-chain corrupted"); + DBUG_RETURN(-1); + } + + if (!(param->testflag & T_SILENT)) puts("- check index reference"); + + all_keydata=all_totaldata=key_totlength=0; + old_record_checksum=0; + init_checksum=param->record_checksum; + if (!(share->options & + (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD))) + old_record_checksum=calc_checksum(info->state->records+info->state->del-1)* + share->base.pack_reclength; + rec_per_key_part= param->rec_per_key_part; + for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ; + rec_per_key_part+=keyinfo->keysegs, key++, keyinfo++) + { + param->key_crc[key]=0; + if (! mi_is_key_active(share->state.key_map, key)) + { + /* Remember old statistics for key */ + memcpy((char*) rec_per_key_part, + (char*) (share->state.rec_per_key_part + + (uint) (rec_per_key_part - param->rec_per_key_part)), + keyinfo->keysegs*sizeof(*rec_per_key_part)); + continue; + } + found_keys++; + + param->record_checksum=init_checksum; + + bzero((char*) ¶m->unique_count,sizeof(param->unique_count)); + bzero((char*) ¶m->notnull_count,sizeof(param->notnull_count)); + + if ((!(param->testflag & T_SILENT))) + printf ("- check data record references index: %d\n",key+1); + if (keyinfo->flag & (HA_FULLTEXT | HA_SPATIAL)) + full_text_keys++; + if (share->state.key_root[key] == HA_OFFSET_ERROR && + (info->state->records == 0 || keyinfo->flag & HA_FULLTEXT)) + goto do_stat; + if (!_mi_fetch_keypage(info,keyinfo,share->state.key_root[key], + DFLT_INIT_HITS,info->buff,0)) + { + mi_check_print_error(param,"Can't read indexpage from filepos: %s", + llstr(share->state.key_root[key],buff)); + if (!(param->testflag & T_INFO)) + DBUG_RETURN(-1); + result= -1; + continue; + } + param->key_file_blocks+=keyinfo->block_length; + keys=0; + param->keydata=param->totaldata=0; + param->key_blocks=0; + param->max_level=0; + if (chk_index(param,info,keyinfo,share->state.key_root[key],info->buff, + &keys, param->key_crc+key,1)) + DBUG_RETURN(-1); + if(!(keyinfo->flag & (HA_FULLTEXT | HA_SPATIAL))) + { + if (keys != info->state->records) + { + mi_check_print_error(param,"Found %s keys of %s",llstr(keys,buff), + llstr(info->state->records,buff2)); + if (!(param->testflag & T_INFO)) + DBUG_RETURN(-1); + result= -1; + continue; + } + if (found_keys - full_text_keys == 1 && + ((share->options & + (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) || + (param->testflag & T_DONT_CHECK_CHECKSUM))) + old_record_checksum=param->record_checksum; + else if (old_record_checksum != param->record_checksum) + { + if (key) + mi_check_print_error(param,"Key %u doesn't point at same records that key 1", + key+1); + else + mi_check_print_error(param,"Key 1 doesn't point at all records"); + if (!(param->testflag & T_INFO)) + DBUG_RETURN(-1); + result= -1; + continue; + } + } + if ((uint) share->base.auto_key -1 == key) + { + /* Check that auto_increment key is bigger than max key value */ + ulonglong auto_increment; + info->lastinx=key; + _mi_read_key_record(info, 0L, info->rec_buff); + auto_increment= retrieve_auto_increment(info, info->rec_buff); + if (auto_increment > info->s->state.auto_increment) + { + mi_check_print_warning(param, "Auto-increment value: %s is smaller " + "than max used value: %s", + llstr(info->s->state.auto_increment,buff2), + llstr(auto_increment, buff)); + } + if (param->testflag & T_AUTO_INC) + { + set_if_bigger(info->s->state.auto_increment, + auto_increment); + set_if_bigger(info->s->state.auto_increment, + param->auto_increment_value); + } + + /* Check that there isn't a row with auto_increment = 0 in the table */ + mi_extra(info,HA_EXTRA_KEYREAD,0); + bzero(info->lastkey,keyinfo->seg->length); + if (!mi_rkey(info, info->rec_buff, key, (const uchar*) info->lastkey, + (key_part_map)1, HA_READ_KEY_EXACT)) + { + /* Don't count this as a real warning, as myisamchk can't correct it */ + my_bool save=param->warning_printed; + mi_check_print_warning(param, "Found row where the auto_increment " + "column has the value 0"); + param->warning_printed=save; + } + mi_extra(info,HA_EXTRA_NO_KEYREAD,0); + } + + length=(my_off_t) isam_key_length(info,keyinfo)*keys + param->key_blocks*2; + if (param->testflag & T_INFO && param->totaldata != 0L && keys != 0L) + printf("Key: %2d: Keyblocks used: %3d%% Packed: %4d%% Max levels: %2d\n", + key+1, + (int) (my_off_t2double(param->keydata)*100.0/my_off_t2double(param->totaldata)), + (int) ((my_off_t2double(length) - my_off_t2double(param->keydata))*100.0/ + my_off_t2double(length)), + param->max_level); + all_keydata+=param->keydata; all_totaldata+=param->totaldata; key_totlength+=length; + +do_stat: + if (param->testflag & T_STATISTICS) + update_key_parts(keyinfo, rec_per_key_part, param->unique_count, + param->stats_method == MI_STATS_METHOD_IGNORE_NULLS? + param->notnull_count: NULL, + (ulonglong)info->state->records); + } + if (param->testflag & T_INFO) + { + if (all_totaldata != 0L && found_keys > 0) + printf("Total: Keyblocks used: %3d%% Packed: %4d%%\n\n", + (int) (my_off_t2double(all_keydata)*100.0/ + my_off_t2double(all_totaldata)), + (int) ((my_off_t2double(key_totlength) - + my_off_t2double(all_keydata))*100.0/ + my_off_t2double(key_totlength))); + else if (all_totaldata != 0L && mi_is_any_key_active(share->state.key_map)) + puts(""); + } + if (param->key_file_blocks != info->state->key_file_length && + param->keys_in_use != ~(ulonglong) 0) + mi_check_print_warning(param, "Some data are unreferenced in keyfile"); + if (found_keys != full_text_keys) + param->record_checksum=old_record_checksum-init_checksum; /* Remove delete links */ + else + param->record_checksum=0; + DBUG_RETURN(result); +} /* chk_key */ + + +static int chk_index_down(HA_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo, + my_off_t page, uchar *buff, ha_rows *keys, + ha_checksum *key_checksum, uint level) +{ + char llbuff[22],llbuff2[22]; + DBUG_ENTER("chk_index_down"); + + /* Key blocks must lay within the key file length entirely. */ + if (page + keyinfo->block_length > info->state->key_file_length) + { + /* purecov: begin tested */ + /* Give it a chance to fit in the real file size. */ + my_off_t max_length= mysql_file_seek(info->s->kfile, 0L, MY_SEEK_END, + MYF(MY_THREADSAFE)); + mi_check_print_error(param, "Invalid key block position: %s " + "key block size: %u file_length: %s", + llstr(page, llbuff), keyinfo->block_length, + llstr(info->state->key_file_length, llbuff2)); + if (page + keyinfo->block_length > max_length) + goto err; + /* Fix the remebered key file length. */ + info->state->key_file_length= (max_length & + ~ (my_off_t) (keyinfo->block_length - 1)); + /* purecov: end */ + } + + /* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */ + if (page & (MI_MIN_KEY_BLOCK_LENGTH - 1)) + { + /* purecov: begin tested */ + mi_check_print_error(param, "Mis-aligned key block: %s " + "minimum key block length: %u", + llstr(page, llbuff), MI_MIN_KEY_BLOCK_LENGTH); + goto err; + /* purecov: end */ + } + + if (!_mi_fetch_keypage(info,keyinfo,page, DFLT_INIT_HITS,buff,0)) + { + mi_check_print_error(param,"Can't read key from filepos: %s", + llstr(page,llbuff)); + goto err; + } + param->key_file_blocks+=keyinfo->block_length; + if (chk_index(param,info,keyinfo,page,buff,keys,key_checksum,level)) + goto err; + + DBUG_RETURN(0); + + /* purecov: begin tested */ +err: + DBUG_RETURN(1); + /* purecov: end */ +} + + +/* + "Ignore NULLs" statistics collection method: process first index tuple. + + SYNOPSIS + mi_collect_stats_nonulls_first() + keyseg IN Array of key part descriptions + notnull INOUT Array, notnull[i] = (number of {keypart1...keypart_i} + tuples that don't contain NULLs) + key IN Key values tuple + + DESCRIPTION + Process the first index tuple - find out which prefix tuples don't + contain NULLs, and update the array of notnull counters accordingly. +*/ + +static +void mi_collect_stats_nonulls_first(HA_KEYSEG *keyseg, ulonglong *notnull, + uchar *key) +{ + uint first_null, kp; + first_null= (uint) (ha_find_null(keyseg, key) - keyseg); + /* + All prefix tuples that don't include keypart_{first_null} are not-null + tuples (and all others aren't), increment counters for them. + */ + for (kp= 0; kp < first_null; kp++) + notnull[kp]++; +} + + +/* + "Ignore NULLs" statistics collection method: process next index tuple. + + SYNOPSIS + mi_collect_stats_nonulls_next() + keyseg IN Array of key part descriptions + notnull INOUT Array, notnull[i] = (number of {keypart1...keypart_i} + tuples that don't contain NULLs) + prev_key IN Previous key values tuple + last_key IN Next key values tuple + + DESCRIPTION + Process the next index tuple: + 1. Find out which prefix tuples of last_key don't contain NULLs, and + update the array of notnull counters accordingly. + 2. Find the first keypart number where the prev_key and last_key tuples + are different(A), or last_key has NULL value(B), and return it, so the + caller can count number of unique tuples for each key prefix. We don't + need (B) to be counted, and that is compensated back in + update_key_parts(). + + RETURN + 1 + number of first keypart where values differ or last_key tuple has NULL +*/ + +static +int mi_collect_stats_nonulls_next(HA_KEYSEG *keyseg, ulonglong *notnull, + uchar *prev_key, uchar *last_key) +{ + uint diffs[2]; + uint first_null_seg, kp; + HA_KEYSEG *seg; + + /* + Find the first keypart where values are different or either of them is + NULL. We get results in diffs array: + diffs[0]= 1 + number of first different keypart + diffs[1]=offset: (last_key + diffs[1]) points to first value in + last_key that is NULL or different from corresponding + value in prev_key. + */ + ha_key_cmp(keyseg, prev_key, last_key, USE_WHOLE_KEY, + SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diffs); + seg= keyseg + diffs[0] - 1; + + /* Find first NULL in last_key */ + first_null_seg= (uint) (ha_find_null(seg, last_key + diffs[1]) - keyseg); + for (kp= 0; kp < first_null_seg; kp++) + notnull[kp]++; + + /* + Return 1+ number of first key part where values differ. Don't care if + these were NULLs and not .... We compensate for that in + update_key_parts. + */ + return diffs[0]; +} + + + /* Check if index is ok */ + +static int chk_index(HA_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo, + my_off_t page, uchar *buff, ha_rows *keys, + ha_checksum *key_checksum, uint level) +{ + int flag; + uint used_length,comp_flag,nod_flag,key_length=0; + uchar key[HA_MAX_POSSIBLE_KEY_BUFF],*temp_buff,*keypos,*old_keypos,*endpos; + my_off_t next_page,record; + char llbuff[22]; + uint diff_pos[2]; + DBUG_ENTER("chk_index"); + DBUG_DUMP("buff",(uchar*) buff,mi_getint(buff)); + + /* TODO: implement appropriate check for RTree keys */ + if (keyinfo->flag & HA_SPATIAL) + DBUG_RETURN(0); + + if (!(temp_buff=(uchar*) my_alloca((uint) keyinfo->block_length))) + { + mi_check_print_error(param,"Not enough memory for keyblock"); + DBUG_RETURN(-1); + } + + if (keyinfo->flag & HA_NOSAME) + { + /* Not real duplicates */ + comp_flag= SEARCH_FIND | SEARCH_UPDATE | SEARCH_INSERT; + } + else + comp_flag=SEARCH_SAME; /* Keys in positionorder */ + nod_flag=mi_test_if_nod(buff); + used_length=mi_getint(buff); + keypos=buff+2+nod_flag; + endpos=buff+used_length; + + param->keydata+=used_length; param->totaldata+=keyinfo->block_length; /* INFO */ + param->key_blocks++; + if (level > param->max_level) + param->max_level=level; + + if (used_length > keyinfo->block_length) + { + mi_check_print_error(param,"Wrong pageinfo at page: %s", + llstr(page,llbuff)); + goto err; + } + for ( ;; ) + { + if (killed_ptr(param)) + goto err; + memcpy((char*) info->lastkey,(char*) key,key_length); + info->lastkey_length=key_length; + if (nod_flag) + { + next_page=_mi_kpos(nod_flag,keypos); + if (chk_index_down(param,info,keyinfo,next_page, + temp_buff,keys,key_checksum,level+1)) + goto err; + } + old_keypos=keypos; + if (keypos >= endpos || + (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0) + break; + if (keypos > endpos) + { + mi_check_print_error(param,"Wrong key block length at page: %s",llstr(page,llbuff)); + goto err; + } + if ((*keys)++ && + (flag=ha_key_cmp(keyinfo->seg,info->lastkey,key,key_length, + comp_flag, diff_pos)) >=0) + { + DBUG_DUMP("old",info->lastkey, info->lastkey_length); + DBUG_DUMP("new",key, key_length); + DBUG_DUMP("new_in_page",old_keypos,(uint) (keypos-old_keypos)); + + if (comp_flag & SEARCH_FIND && flag == 0) + mi_check_print_error(param,"Found duplicated key at page %s",llstr(page,llbuff)); + else + mi_check_print_error(param,"Key in wrong position at page %s",llstr(page,llbuff)); + goto err; + } + if (param->testflag & T_STATISTICS) + { + if (*keys != 1L) /* not first_key */ + { + if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL) + ha_key_cmp(keyinfo->seg,info->lastkey,key,USE_WHOLE_KEY, + SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, + diff_pos); + else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS) + { + diff_pos[0]= mi_collect_stats_nonulls_next(keyinfo->seg, + param->notnull_count, + info->lastkey, key); + } + param->unique_count[diff_pos[0]-1]++; + } + else + { + if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS) + mi_collect_stats_nonulls_first(keyinfo->seg, param->notnull_count, + key); + } + } + (*key_checksum)+= mi_byte_checksum((uchar*) key, + key_length- info->s->rec_reflength); + record= _mi_dpos(info,0,key+key_length); + if (keyinfo->flag & HA_FULLTEXT) /* special handling for ft2 */ + { + uint off; + int subkeys; + get_key_full_length_rdonly(off, key); + subkeys=ft_sintXkorr(key+off); + if (subkeys < 0) + { + ha_rows tmp_keys=0; + if (chk_index_down(param,info,&info->s->ft2_keyinfo,record, + temp_buff,&tmp_keys,key_checksum,1)) + goto err; + if (tmp_keys + subkeys) + { + mi_check_print_error(param, + "Number of words in the 2nd level tree " + "does not match the number in the header. " + "Parent word in on the page %s, offset %u", + llstr(page,llbuff), (uint) (old_keypos-buff)); + goto err; + } + (*keys)+=tmp_keys-1; + continue; + } + /* fall through */ + } + if (record >= info->state->data_file_length) + { +#ifdef DBUG_TRACE + char llbuff2[22], llbuff3[22]; +#endif + mi_check_print_error(param,"Found key at page %s that points to record outside datafile",llstr(page,llbuff)); + DBUG_PRINT("test",("page: %s record: %s filelength: %s", + llstr(page,llbuff),llstr(record,llbuff2), + llstr(info->state->data_file_length,llbuff3))); + DBUG_DUMP("key",key,key_length); + DBUG_DUMP("new_in_page",old_keypos,(uint) (keypos-old_keypos)); + goto err; + } + param->record_checksum+=(ha_checksum) record; + } + if (keypos != endpos) + { + mi_check_print_error(param,"Keyblock size at page %s is not correct. Block length: %d key length: %d", + llstr(page,llbuff), used_length, (keypos - buff)); + goto err; + } + my_afree((uchar*) temp_buff); + DBUG_RETURN(0); + err: + my_afree((uchar*) temp_buff); + DBUG_RETURN(1); +} /* chk_index */ + + + /* Calculate a checksum of 1+2+3+4...N = N*(N+1)/2 without overflow */ + +static ha_checksum calc_checksum(ha_rows count) +{ + ulonglong sum,a,b; + DBUG_ENTER("calc_checksum"); + + sum=0; + a=count; b=count+1; + if (a & 1) + b>>=1; + else + a>>=1; + while (b) + { + if (b & 1) + sum+=a; + a<<=1; b>>=1; + } + DBUG_PRINT("exit",("sum: %lx",(ulong) sum)); + DBUG_RETURN((ha_checksum) sum); +} /* calc_checksum */ + + + /* Calc length of key in normal isam */ + +static uint isam_key_length(MI_INFO *info, register MI_KEYDEF *keyinfo) +{ + uint length; + HA_KEYSEG *keyseg; + DBUG_ENTER("isam_key_length"); + + length= info->s->rec_reflength; + for (keyseg=keyinfo->seg ; keyseg->type ; keyseg++) + length+= keyseg->length; + + DBUG_PRINT("exit",("length: %d",length)); + DBUG_RETURN(length); +} /* key_length */ + + + /* Check that record-link is ok */ + +int chk_data_link(HA_CHECK *param, MI_INFO *info, my_bool extend) +{ + int error,got_error,flag; + uint key, UNINIT_VAR(left_length), b_type; + ha_rows records,del_blocks; + my_off_t used,empty,pos,splits,UNINIT_VAR(start_recpos), + del_length,link_used,start_block; + uchar *record= 0, *UNINIT_VAR(to); + char llbuff[22],llbuff2[22],llbuff3[22]; + ha_checksum intern_record_checksum; + ha_checksum key_checksum[HA_MAX_POSSIBLE_KEY]; + MI_KEYDEF *keyinfo; + MI_BLOCK_INFO block_info; + DBUG_ENTER("chk_data_link"); + + if (!(param->testflag & T_SILENT)) + { + if (extend) + puts("- check records and index references"); + else + puts("- check record links"); + } + + if (!mi_alloc_rec_buff(info, -1, &record)) + { + mi_check_print_error(param,"Not enough memory for record"); + DBUG_RETURN(-1); + } + records=del_blocks=0; + used=link_used=splits=del_length=0; + intern_record_checksum=param->glob_crc=0; + got_error=error=0; + empty=info->s->pack.header_length; + + pos=my_b_tell(¶m->read_cache); + bzero((char*) key_checksum, info->s->base.keys * sizeof(key_checksum[0])); + while (pos < info->state->data_file_length) + { + if (killed_ptr(param)) + goto err2; + switch (info->s->data_file_type) { + case STATIC_RECORD: + if (my_b_read(¶m->read_cache,(uchar*) record, + info->s->base.pack_reclength)) + goto err; + start_recpos=pos; + pos+=info->s->base.pack_reclength; + splits++; + if (*record == '\0') + { + del_blocks++; + del_length+=info->s->base.pack_reclength; + continue; /* Record removed */ + } + param->glob_crc+= (*info->s->calc_check_checksum)(info,record); + used+=info->s->base.pack_reclength; + break; + case DYNAMIC_RECORD: + flag=block_info.second_read=0; + block_info.next_filepos=pos; + do + { + if (_mi_read_cache(¶m->read_cache,(uchar*) block_info.header, + (start_block=block_info.next_filepos), + sizeof(block_info.header), + (flag ? 0 : READING_NEXT) | READING_HEADER)) + goto err; + if (start_block & (MI_DYN_ALIGN_SIZE-1)) + { + mi_check_print_error(param,"Wrong aligned block at %s", + llstr(start_block,llbuff)); + goto err2; + } + b_type=_mi_get_block_info(&block_info,-1,start_block); + if (b_type & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR | + BLOCK_FATAL_ERROR)) + { + if (b_type & BLOCK_SYNC_ERROR) + { + if (flag) + { + mi_check_print_error(param,"Unexpected byte: %d at link: %s", + (int) block_info.header[0], + llstr(start_block,llbuff)); + goto err2; + } + pos=block_info.filepos+block_info.block_len; + goto next; + } + if (b_type & BLOCK_DELETED) + { + if (block_info.block_len < info->s->base.min_block_length) + { + mi_check_print_error(param, + "Deleted block with impossible length %lu at %s", + block_info.block_len,llstr(pos,llbuff)); + goto err2; + } + if ((block_info.next_filepos != HA_OFFSET_ERROR && + block_info.next_filepos >= info->state->data_file_length) || + (block_info.prev_filepos != HA_OFFSET_ERROR && + block_info.prev_filepos >= info->state->data_file_length)) + { + mi_check_print_error(param,"Delete link points outside datafile at %s", + llstr(pos,llbuff)); + goto err2; + } + del_blocks++; + del_length+=block_info.block_len; + pos=block_info.filepos+block_info.block_len; + splits++; + goto next; + } + mi_check_print_error(param,"Wrong bytesec: %d-%d-%d at linkstart: %s", + block_info.header[0],block_info.header[1], + block_info.header[2], + llstr(start_block,llbuff)); + goto err2; + } + if (info->state->data_file_length < block_info.filepos+ + block_info.block_len) + { + mi_check_print_error(param, + "Recordlink that points outside datafile at %s", + llstr(pos,llbuff)); + got_error=1; + break; + } + splits++; + if (!flag++) /* First block */ + { + start_recpos=pos; + pos=block_info.filepos+block_info.block_len; + if (block_info.rec_len > (uint) info->s->base.max_pack_length) + { + mi_check_print_error(param,"Found too long record (%lu) at %s", + (ulong) block_info.rec_len, + llstr(start_recpos,llbuff)); + got_error=1; + break; + } + if (info->s->base.blobs) + { + if (!(to= mi_alloc_rec_buff(info, block_info.rec_len, + &info->rec_buff))) + { + mi_check_print_error(param, + "Not enough memory (%lu) for blob at %s", + (ulong) block_info.rec_len, + llstr(start_recpos,llbuff)); + got_error=1; + break; + } + } + else + to= info->rec_buff; + left_length=block_info.rec_len; + } + if (left_length < block_info.data_len) + { + mi_check_print_error(param,"Found too long record (%lu) at %s", + (ulong) block_info.data_len, + llstr(start_recpos,llbuff)); + got_error=1; + break; + } + if (_mi_read_cache(¶m->read_cache,(uchar*) to,block_info.filepos, + (uint) block_info.data_len, + flag == 1 ? READING_NEXT : 0)) + goto err; + to+=block_info.data_len; + link_used+= block_info.filepos-start_block; + used+= block_info.filepos - start_block + block_info.data_len; + empty+=block_info.block_len-block_info.data_len; + left_length-=block_info.data_len; + if (left_length) + { + if (b_type & BLOCK_LAST) + { + mi_check_print_error(param, + "Wrong record length %s of %s at %s", + llstr(block_info.rec_len-left_length,llbuff), + llstr(block_info.rec_len, llbuff2), + llstr(start_recpos,llbuff3)); + got_error=1; + break; + } + if (info->state->data_file_length < block_info.next_filepos) + { + mi_check_print_error(param, + "Found next-recordlink that points outside datafile at %s", + llstr(block_info.filepos,llbuff)); + got_error=1; + break; + } + } + } while (left_length); + if (! got_error) + { + if (_mi_rec_unpack(info,record,info->rec_buff,block_info.rec_len) == + MY_FILE_ERROR) + { + mi_check_print_error(param,"Found wrong record at %s", + llstr(start_recpos,llbuff)); + got_error=1; + } + else + { + info->checksum= (*info->s->calc_check_checksum)(info,record); + if (param->testflag & (T_EXTEND | T_MEDIUM | T_VERBOSE)) + { + if (_mi_rec_check(info,record, info->rec_buff,block_info.rec_len, + MY_TEST(info->s->calc_checksum))) + { + mi_check_print_error(param,"Found wrong packed record at %s", + llstr(start_recpos,llbuff)); + got_error=1; + } + } + if (!got_error) + param->glob_crc+= info->checksum; + } + } + else if (!flag) + pos=block_info.filepos+block_info.block_len; + break; + case COMPRESSED_RECORD: + if (_mi_read_cache(¶m->read_cache,(uchar*) block_info.header, pos, + info->s->pack.ref_length, READING_NEXT)) + goto err; + start_recpos=pos; + splits++; + (void) _mi_pack_get_block_info(info, &info->bit_buff, &block_info, + &info->rec_buff, -1, start_recpos); + pos=block_info.filepos+block_info.rec_len; + if (block_info.rec_len < (uint) info->s->min_pack_length || + block_info.rec_len > (uint) info->s->max_pack_length) + { + mi_check_print_error(param, + "Found block with wrong recordlength: %d at %s", + block_info.rec_len, llstr(start_recpos,llbuff)); + got_error=1; + break; + } + if (_mi_read_cache(¶m->read_cache,(uchar*) info->rec_buff, + block_info.filepos, block_info.rec_len, READING_NEXT)) + goto err; + info->rec_buff[block_info.rec_len]= 0; /* Keep valgrind happy */ + if (_mi_pack_rec_unpack(info, &info->bit_buff, record, + info->rec_buff, block_info.rec_len)) + { + mi_check_print_error(param,"Found wrong record at %s", + llstr(start_recpos,llbuff)); + got_error=1; + } + param->glob_crc+= (*info->s->calc_check_checksum)(info,record); + link_used+= (block_info.filepos - start_recpos); + used+= (pos-start_recpos); + break; + default: + DBUG_ASSERT(0); /* Impossible */ + break; + } /* switch */ + if (param->fix_record) + param->fix_record(info, record, -1); + if (! got_error) + { + intern_record_checksum+=(ha_checksum) start_recpos; + records++; + if (param->testflag & T_WRITE_LOOP && records % WRITE_COUNT == 0) + { + printf("%s\r", llstr(records,llbuff)); (void) fflush(stdout); + } + + /* Check if keys match the record */ + + for (key=0,keyinfo= info->s->keyinfo; key < info->s->base.keys; + key++,keyinfo++) + { + if (mi_is_key_active(info->s->state.key_map, key)) + { + if(!(keyinfo->flag & HA_FULLTEXT)) + { + uint key_length=_mi_make_key(info,key,info->lastkey,record, + start_recpos); + if (extend) + { + /* We don't need to lock the key tree here as we don't allow + concurrent threads when running myisamchk + */ + int search_result= +#ifdef HAVE_RTREE_KEYS + (keyinfo->flag & HA_SPATIAL) ? + rtree_find_first(info, key, info->lastkey, key_length, + MBR_EQUAL | MBR_DATA) : +#endif + _mi_search(info,keyinfo,info->lastkey,key_length, + SEARCH_SAME, info->s->state.key_root[key]); + if (search_result) + { + mi_check_print_error(param,"Record at: %10s " + "Can't find key for index: %2d", + llstr(start_recpos,llbuff),key+1); + if (error++ > MAXERR || !(param->testflag & T_VERBOSE)) + goto err2; + } + } + else + key_checksum[key]+=mi_byte_checksum((uchar*) info->lastkey, + key_length); + } + } + } + } + else + { + got_error=0; + if (error++ > MAXERR || !(param->testflag & T_VERBOSE)) + goto err2; + } + next:; /* Next record */ + } + if (param->testflag & T_WRITE_LOOP) + { + (void) fputs(" \r",stdout); (void) fflush(stdout); + } + if (records != info->state->records) + { + mi_check_print_error(param,"Record-count is not ok; is %-10s Should be: %s", + llstr(records,llbuff), llstr(info->state->records,llbuff2)); + error=1; + } + else if (param->record_checksum && + param->record_checksum != intern_record_checksum) + { + mi_check_print_error(param, + "Keypointers and record positions doesn't match"); + error=1; + } + else if (param->glob_crc != info->state->checksum && + (info->s->options & + (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))) + { + mi_check_print_warning(param, + "Record checksum is not the same as checksum stored in the index file\n"); + error=1; + } + else if (!extend) + { + for (key=0 ; key < info->s->base.keys; key++) + { + if (key_checksum[key] != param->key_crc[key] && + !(info->s->keyinfo[key].flag & (HA_FULLTEXT | HA_SPATIAL))) + { + mi_check_print_error(param,"Checksum for key: %2d doesn't match checksum for records", + key+1); + error=1; + } + } + } + + if (del_length != info->state->empty) + { + mi_check_print_warning(param, + "Found %s deleted space. Should be %s", + llstr(del_length,llbuff2), + llstr(info->state->empty,llbuff)); + } + if (used+empty+del_length != info->state->data_file_length) + { + mi_check_print_warning(param, + "Found %s record-data and %s unused data and %s deleted-data", + llstr(used,llbuff),llstr(empty,llbuff2), + llstr(del_length,llbuff3)); + mi_check_print_warning(param, + "Total %s, Should be: %s", + llstr((used+empty+del_length),llbuff), + llstr(info->state->data_file_length,llbuff2)); + } + if (del_blocks != info->state->del) + { + mi_check_print_warning(param, + "Found %10s deleted blocks Should be: %s", + llstr(del_blocks,llbuff), + llstr(info->state->del,llbuff2)); + } + if (splits != info->s->state.split) + { + mi_check_print_warning(param, + "Found %10s key parts. Should be: %s", + llstr(splits,llbuff), + llstr(info->s->state.split,llbuff2)); + } + if (param->testflag & T_INFO) + { + if (param->warning_printed || param->error_printed || param->note_printed) + puts(""); + if (used != 0 && ! param->error_printed) + { + printf("Records:%18s M.recordlength:%9lu Packed:%14.0f%%\n", + llstr(records,llbuff), (long)((used-link_used)/records), + (info->s->base.blobs ? 0.0 : + (ulonglong2double((ulonglong) info->s->base.reclength*records)- + my_off_t2double(used))/ + ulonglong2double((ulonglong) info->s->base.reclength*records)*100.0)); + printf("Recordspace used:%9.0f%% Empty space:%12d%% Blocks/Record: %6.2f\n", + (ulonglong2double(used-link_used)/ulonglong2double(used-link_used+empty)*100.0), + (!records ? 100 : (int) (ulonglong2double(del_length+empty)/ + my_off_t2double(used)*100.0)), + ulonglong2double(splits - del_blocks) / records); + } + printf("Record blocks:%12s Delete blocks:%10s\n", + llstr(splits-del_blocks,llbuff),llstr(del_blocks,llbuff2)); + printf("Record data: %12s Deleted data: %10s\n", + llstr(used-link_used,llbuff),llstr(del_length,llbuff2)); + printf("Lost space: %12s Linkdata: %10s\n", + llstr(empty,llbuff),llstr(link_used,llbuff2)); + } + my_free(mi_get_rec_buff_ptr(info, record)); + DBUG_RETURN (error); + err: + mi_check_print_error(param,"got error: %d when reading datafile at record: %s",my_errno, llstr(records,llbuff)); + err2: + my_free(mi_get_rec_buff_ptr(info, record)); + param->testflag|=T_RETRY_WITHOUT_QUICK; + DBUG_RETURN(1); +} /* chk_data_link */ + + +/** + @brief Drop all indexes + + @param[in] param check parameters + @param[in] info MI_INFO handle + @param[in] force if to force drop all indexes + + @return status + @retval 0 OK + @retval != 0 Error + + @note + Once allocated, index blocks remain part of the key file forever. + When indexes are disabled, no block is freed. When enabling indexes, + no block is freed either. The new indexes are create from new + blocks. (Bug #4692) + + Before recreating formerly disabled indexes, the unused blocks + must be freed. There are two options to do this: + - Follow the tree of disabled indexes, add all blocks to the + deleted blocks chain. Would require a lot of random I/O. + - Drop all blocks by clearing all index root pointers and all + delete chain pointers and resetting key_file_length to the end + of the index file header. This requires to recreate all indexes, + even those that may still be intact. + The second method is probably faster in most cases. + + When disabling indexes, MySQL disables either all indexes or all + non-unique indexes. When MySQL [re-]enables disabled indexes + (T_CREATE_MISSING_KEYS), then we either have "lost" blocks in the + index file, or there are no non-unique indexes. In the latter case, + mi_repair*() would not be called as there would be no disabled + indexes. + + If there would be more unique indexes than disabled (non-unique) + indexes, we could do the first method. But this is not implemented + yet. By now we drop and recreate all indexes when repair is called. + + However, there is an exception. Sometimes MySQL disables non-unique + indexes when the table is empty (e.g. when copying a table in + mysql_alter_table()). When enabling the non-unique indexes, they + are still empty. So there is no index block that can be lost. This + optimization is implemented in this function. + + Note that in normal repair (T_CREATE_MISSING_KEYS not set) we + recreate all enabled indexes unconditonally. We do not change the + key_map. Otherwise we invert the key map temporarily (outside of + this function) and recreate the then "seemingly" enabled indexes. + When we cannot use the optimization, and drop all indexes, we + pretend that all indexes were disabled. By the inversion, we will + then recrate all indexes. +*/ + +static int mi_drop_all_indexes(HA_CHECK *param, MI_INFO *info, my_bool force) +{ + MYISAM_SHARE *share= info->s; + MI_STATE_INFO *state= &share->state; + uint i; + int error; + DBUG_ENTER("mi_drop_all_indexes"); + + /* + If any of the disabled indexes has a key block assigned, we must + drop and recreate all indexes to avoid losing index blocks. + + If we want to recreate disabled indexes only _and_ all of these + indexes are empty, we don't need to recreate the existing indexes. + */ + if (!force && (param->testflag & T_CREATE_MISSING_KEYS)) + { + DBUG_PRINT("repair", ("creating missing indexes")); + for (i= 0; i < share->base.keys; i++) + { + DBUG_PRINT("repair", ("index #: %u key_root: 0x%lx active: %d", + i, (long) state->key_root[i], + mi_is_key_active(state->key_map, i))); + if ((state->key_root[i] != HA_OFFSET_ERROR) && + !mi_is_key_active(state->key_map, i)) + { + /* + This index has at least one key block and it is disabled. + We would lose its block(s) if would just recreate it. + So we need to drop and recreate all indexes. + */ + DBUG_PRINT("repair", ("nonempty and disabled: recreate all")); + break; + } + } + if (i >= share->base.keys) + { + /* + All of the disabled indexes are empty. We can just recreate them. + Flush dirty blocks of this index file from key cache and remove + all blocks of this index file from key cache. + */ + DBUG_PRINT("repair", ("all disabled are empty: create missing")); + error= flush_key_blocks(share->key_cache, share->kfile, + &share->dirty_part_map, + FLUSH_FORCE_WRITE); + goto end; + } + /* + We do now drop all indexes and declare them disabled. With the + T_CREATE_MISSING_KEYS flag, mi_repair*() will recreate all + disabled indexes and enable them. + */ + mi_clear_all_keys_active(state->key_map); + DBUG_PRINT("repair", ("declared all indexes disabled")); + } + + /* Remove all key blocks of this index file from key cache. */ + if ((error= flush_key_blocks(share->key_cache, share->kfile, + &share->dirty_part_map, + FLUSH_IGNORE_CHANGED))) + goto end; /* purecov: inspected */ + + /* Clear index root block pointers. */ + for (i= 0; i < share->base.keys; i++) + state->key_root[i]= HA_OFFSET_ERROR; + + /* Clear the delete chains. */ + for (i= 0; i < state->header.max_block_size_index; i++) + state->key_del[i]= HA_OFFSET_ERROR; + + /* Reset index file length to end of index file header. */ + info->state->key_file_length= share->base.keystart; + + DBUG_PRINT("repair", ("dropped all indexes")); + /* error= 0; set by last (error= flush_key_bocks()). */ + + end: + DBUG_RETURN(error); +} + + + /* Recover old table by reading each record and writing all keys */ + /* Save new datafile-name in temp_filename */ + +int mi_repair(HA_CHECK *param, register MI_INFO *info, + char * name, int rep_quick) +{ + int error,got_error; + ha_rows start_records,new_header_length; + my_off_t del; + File new_file; + MYISAM_SHARE *share=info->s; + char llbuff[22],llbuff2[22]; + MI_SORT_INFO sort_info; + MI_SORT_PARAM sort_param; + DBUG_ENTER("mi_repair"); + + bzero((char *)&sort_info, sizeof(sort_info)); + bzero((char *)&sort_param, sizeof(sort_param)); + start_records=info->state->records; + new_header_length=(param->testflag & T_UNPACK) ? 0L : + share->pack.header_length; + got_error=1; + new_file= -1; + sort_param.sort_info=&sort_info; + param->retry_repair= 0; + param->warning_printed= param->note_printed= 0; + param->error_printed= 0; + + if (!(param->testflag & T_SILENT)) + { + printf("- recovering (with keycache) MyISAM-table '%s'\n",name); + printf("Data records: %s\n", llstr(info->state->records,llbuff)); + } + param->testflag|=T_REP; /* for easy checking */ + + if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD)) + param->testflag|=T_CALC_CHECKSUM; + + DBUG_ASSERT(param->use_buffers < SIZE_T_MAX); + + if (!param->using_global_keycache) + (void) init_key_cache(dflt_key_cache, param->key_cache_block_size, + (size_t) param->use_buffers, 0, 0, 0, 0); + + if (init_io_cache(¶m->read_cache,info->dfile, + (uint) param->read_buffer_length, + READ_CACHE,share->pack.header_length,1,MYF(MY_WME))) + { + bzero(&info->rec_cache,sizeof(info->rec_cache)); + goto err; + } + info->opt_flag|=WRITE_CACHE_USED; + if (!mi_alloc_rec_buff(info, -1, &sort_param.record) || + !mi_alloc_rec_buff(info, -1, &sort_param.rec_buff)) + { + mi_check_print_error(param, "Not enough memory for extra record"); + goto err; + } + + if (!rep_quick) + { + /* Get real path for data file */ + if ((new_file= mysql_file_create(mi_key_file_datatmp, + fn_format(param->temp_filename, + share->data_file_name, "", + DATA_TMP_EXT, 2+4), + 0, param->tmpfile_createflag, + MYF(0))) < 0) + { + mi_check_print_error(param,"Can't create new tempfile: '%s'", + param->temp_filename); + goto err; + } + if (new_header_length && + filecopy(param,new_file,info->dfile,0L,new_header_length, + "datafile-header")) + goto err; + info->s->state.dellink= HA_OFFSET_ERROR; + if (param->testflag & T_UNPACK) + { + share->options&= ~HA_OPTION_COMPRESS_RECORD; + mi_int2store(share->state.header.options,share->options); + } + if (init_io_cache(&info->rec_cache, new_file, + (uint) param->write_buffer_length, + WRITE_CACHE, new_header_length, 1, + MYF(MY_WME | MY_WAIT_IF_FULL))) + goto err; + } + sort_info.info=info; + sort_info.param = param; + sort_param.read_cache=param->read_cache; + sort_param.pos=sort_param.max_pos=share->pack.header_length; + sort_param.filepos=new_header_length; + param->read_cache.end_of_file=sort_info.filelength= + mysql_file_seek(info->dfile, 0L, MY_SEEK_END, MYF(0)); + if (info->state->data_file_length == 0) + info->state->data_file_length= sort_info.filelength; + sort_info.dupp=0; + sort_param.fix_datafile= (my_bool) (! rep_quick); + sort_param.master=1; + sort_info.max_records= ~(ha_rows) 0; + + set_data_file_type(&sort_info, share); + del=info->state->del; + info->state->records=info->state->del=share->state.split=0; + info->state->empty=0; + param->glob_crc=0; + if (param->testflag & T_CALC_CHECKSUM) + sort_param.calc_checksum= 1; + + info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + + /* This function always recreates all enabled indexes. */ + if (param->testflag & T_CREATE_MISSING_KEYS) + mi_set_all_keys_active(share->state.key_map, share->base.keys); + mi_drop_all_indexes(param, info, TRUE); + + lock_memory(param); /* Everything is alloced */ + + /* Re-create all keys, which are set in key_map. */ + while (!(error=sort_get_next_record(&sort_param))) + { + if (writekeys(&sort_param)) + { + if (my_errno != HA_ERR_FOUND_DUPP_KEY) + goto err; + DBUG_DUMP("record",(uchar*) sort_param.record,share->base.pack_reclength); + mi_check_print_info(param,"Duplicate key %2d for record at %10s against new record at %10s", + info->errkey+1, + llstr(sort_param.start_recpos,llbuff), + llstr(info->dupp_key_pos,llbuff2)); + if (param->testflag & T_VERBOSE) + { + (void) _mi_make_key(info,(uint) info->errkey,info->lastkey, + sort_param.record,0L); + _mi_print_key(stdout,share->keyinfo[info->errkey].seg,info->lastkey, + USE_WHOLE_KEY); + } + sort_info.dupp++; + if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK) + { + param->testflag|=T_RETRY_WITHOUT_QUICK; + param->error_printed=1; + goto err; + } + continue; + } + if (sort_write_record(&sort_param)) + goto err; + } + if (error > 0 || write_data_suffix(&sort_info, (my_bool)!rep_quick) || + flush_io_cache(&info->rec_cache) || param->read_cache.error < 0) + goto err; + + if (param->testflag & T_WRITE_LOOP) + { + (void) fputs(" \r",stdout); (void) fflush(stdout); + } + if (mysql_file_chsize(share->kfile, info->state->key_file_length, 0, MYF(0))) + { + mi_check_print_warning(param, + "Can't change size of indexfile, error: %d", + my_errno); + goto err; + } + + if (rep_quick && del+sort_info.dupp != info->state->del) + { + mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records"); + mi_check_print_error(param,"Run recovery again without --quick"); + got_error=1; + param->retry_repair=1; + param->testflag|=T_RETRY_WITHOUT_QUICK; + goto err; + } + if (param->testflag & T_SAFE_REPAIR) + { + /* Don't repair if we loosed more than one row */ + if (info->state->records+1 < start_records) + { + info->state->records=start_records; + got_error=1; + goto err; + } + } + + if (!rep_quick) + { + mysql_file_close(info->dfile, MYF(0)); + info->dfile=new_file; + info->state->data_file_length=sort_param.filepos; + share->state.version=(ulong) time((time_t*) 0); /* Force reopen */ + } + else + { + info->state->data_file_length=sort_param.max_pos; + } + if (param->testflag & T_CALC_CHECKSUM) + info->state->checksum=param->glob_crc; + + if (!(param->testflag & T_SILENT)) + { + if (start_records != info->state->records) + printf("Data records: %s\n", llstr(info->state->records,llbuff)); + if (sort_info.dupp) + mi_check_print_warning(param, + "%s records have been removed", + llstr(sort_info.dupp,llbuff)); + } + + got_error=0; + /* If invoked by external program that uses thr_lock */ + if (&share->state.state != info->state) + memcpy( &share->state.state, info->state, sizeof(*info->state)); + +err: + if (!got_error) + { + /* Replace the actual file with the temporary file */ + if (new_file >= 0) + { + got_error= replace_data_file(param, info, new_file); + new_file= -1; + param->retry_repair= 0; + } + } + if (got_error) + { + if (! param->error_printed) + mi_check_print_error(param,"%d for record at pos %s",my_errno, + llstr(sort_param.start_recpos,llbuff)); + if (new_file >= 0) + { + (void) mysql_file_close(new_file, MYF(0)); + (void) mysql_file_delete(mi_key_file_datatmp, + param->temp_filename, MYF(MY_WME)); + info->rec_cache.file=-1; /* don't flush data to new_file, it's closed */ + } + mi_mark_crashed_on_repair(info); + } + my_free(mi_get_rec_buff_ptr(info, sort_param.rec_buff)); + my_free(mi_get_rec_buff_ptr(info, sort_param.record)); + my_free(sort_info.buff); + (void) end_io_cache(¶m->read_cache); + info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED); + (void) end_io_cache(&info->rec_cache); + got_error|=flush_blocks(param, share->key_cache, share->kfile, + &share->dirty_part_map); + if (!got_error && param->testflag & T_UNPACK) + { + share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD; + share->pack.header_length=0; + share->data_file_type=sort_info.new_data_file_type; + } + share->state.changed|= (STATE_NOT_OPTIMIZED_KEYS | STATE_NOT_SORTED_PAGES | + STATE_NOT_ANALYZED); + DBUG_RETURN(got_error); +} + + +/* Uppate keyfile when doing repair */ + +static int writekeys(MI_SORT_PARAM *sort_param) +{ + register uint i; + uchar *key; + MI_INFO *info= sort_param->sort_info->info; + uchar *buff= sort_param->record; + my_off_t filepos= sort_param->filepos; + DBUG_ENTER("writekeys"); + + key=info->lastkey+info->s->base.max_key_length; + for (i=0 ; i < info->s->base.keys ; i++) + { + if (mi_is_key_active(info->s->state.key_map, i)) + { + if (info->s->keyinfo[i].flag & HA_FULLTEXT ) + { + if (_mi_ft_add(info, i, key, buff, filepos)) + goto err; + } +#ifdef HAVE_SPATIAL + else if (info->s->keyinfo[i].flag & HA_SPATIAL) + { + uint key_length=_mi_make_key(info,i,key,buff,filepos); + if (rtree_insert(info, i, key, key_length)) + goto err; + } +#endif /*HAVE_SPATIAL*/ + else + { + uint key_length=_mi_make_key(info,i,key,buff,filepos); + if (_mi_ck_write(info,i,key,key_length)) + goto err; + } + } + } + DBUG_RETURN(0); + + err: + if (my_errno == HA_ERR_FOUND_DUPP_KEY) + { + info->errkey=(int) i; /* This key was found */ + while ( i-- > 0 ) + { + if (mi_is_key_active(info->s->state.key_map, i)) + { + if (info->s->keyinfo[i].flag & HA_FULLTEXT) + { + if (_mi_ft_del(info,i, key,buff,filepos)) + break; + } + else + { + uint key_length=_mi_make_key(info,i,key,buff,filepos); + if (_mi_ck_delete(info,i,key,key_length)) + break; + } + } + } + } + /* Remove checksum that was added to glob_crc in sort_get_next_record */ + if (sort_param->calc_checksum) + sort_param->sort_info->param->glob_crc-= info->checksum; + DBUG_PRINT("error",("errno: %d",my_errno)); + DBUG_RETURN(-1); +} /* writekeys */ + + + /* Change all key-pointers that points to a records */ + +int movepoint(register MI_INFO *info, uchar *record, my_off_t oldpos, + my_off_t newpos, uint prot_key) +{ + register uint i; + uchar *key; + uint key_length; + DBUG_ENTER("movepoint"); + + key=info->lastkey+info->s->base.max_key_length; + for (i=0 ; i < info->s->base.keys; i++) + { + if (i != prot_key && mi_is_key_active(info->s->state.key_map, i)) + { + key_length=_mi_make_key(info,i,key,record,oldpos); + if (info->s->keyinfo[i].flag & HA_NOSAME) + { /* Change pointer direct */ + uint nod_flag; + MI_KEYDEF *keyinfo; + keyinfo=info->s->keyinfo+i; + if (_mi_search(info,keyinfo,key,USE_WHOLE_KEY, + (uint) (SEARCH_SAME | SEARCH_SAVE_BUFF), + info->s->state.key_root[i])) + DBUG_RETURN(-1); + nod_flag=mi_test_if_nod(info->buff); + _mi_dpointer(info,info->int_keypos-nod_flag- + info->s->rec_reflength,newpos); + if (_mi_write_keypage(info,keyinfo,info->last_keypage, + DFLT_INIT_HITS,info->buff)) + DBUG_RETURN(-1); + } + else + { /* Change old key to new */ + if (_mi_ck_delete(info,i,key,key_length)) + DBUG_RETURN(-1); + key_length=_mi_make_key(info,i,key,record,newpos); + if (_mi_ck_write(info,i,key,key_length)) + DBUG_RETURN(-1); + } + } + } + DBUG_RETURN(0); +} /* movepoint */ + + + /* Tell system that we want all memory for our cache */ + +void lock_memory(HA_CHECK *param __attribute__((unused))) +{ +#ifdef SUN_OS /* Key-cacheing thrases on sun 4.1 */ + if (param->opt_lock_memory) + { + int success = mlockall(MCL_CURRENT); /* or plock(DATLOCK); */ + if (geteuid() == 0 && success != 0) + mi_check_print_warning(param, + "Failed to lock memory. errno %d",my_errno); + } +#endif +} /* lock_memory */ + + + /* Flush all changed blocks to disk */ + +int flush_blocks(HA_CHECK *param, KEY_CACHE *key_cache, File file, + ulonglong *dirty_part_map) +{ + if (flush_key_blocks(key_cache, file, dirty_part_map, FLUSH_RELEASE)) + { + mi_check_print_error(param,"%d when trying to write buffers",my_errno); + return(1); + } + if (!param->using_global_keycache) + end_key_cache(key_cache,1); + return 0; +} /* flush_blocks */ + + + /* Sort index for more efficient reads */ + +int mi_sort_index(HA_CHECK *param, register MI_INFO *info, char * name) +{ + reg2 uint key; + reg1 MI_KEYDEF *keyinfo; + File new_file; + my_off_t index_pos[HA_MAX_POSSIBLE_KEY]; + uint r_locks,w_locks; + int old_lock; + MYISAM_SHARE *share=info->s; + MI_STATE_INFO old_state; + DBUG_ENTER("mi_sort_index"); + + /* cannot sort index files with R-tree indexes */ + for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ; + key++,keyinfo++) + if (keyinfo->key_alg == HA_KEY_ALG_RTREE) + DBUG_RETURN(0); + + if (!(param->testflag & T_SILENT)) + printf("- Sorting index for MyISAM-table '%s'\n",name); + + /* Get real path for index file */ + fn_format(param->temp_filename,name,"", MI_NAME_IEXT,2+4+32); + if ((new_file= mysql_file_create(mi_key_file_datatmp, + fn_format(param->temp_filename, + param->temp_filename, + "", INDEX_TMP_EXT, 2+4), + 0, param->tmpfile_createflag, MYF(0))) < 0) + { + mi_check_print_error(param,"Can't create new tempfile: '%s'", + param->temp_filename); + DBUG_RETURN(-1); + } + if (filecopy(param, new_file,share->kfile,0L, + (ulong) share->base.keystart, "headerblock")) + goto err; + + param->new_file_pos=share->base.keystart; + for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ; + key++,keyinfo++) + { + if (mi_is_key_active(info->s->state.key_map, key) && + share->state.key_root[key] != HA_OFFSET_ERROR) + { + index_pos[key]=param->new_file_pos; /* Write first block here */ + if (sort_one_index(param,info,keyinfo,share->state.key_root[key], + new_file)) + goto err; + } + else + index_pos[key]= HA_OFFSET_ERROR; /* No blocks */ + } + + /* Flush key cache for this file if we are calling this outside myisamchk */ + flush_key_blocks(share->key_cache, share->kfile, &share->dirty_part_map, + FLUSH_IGNORE_CHANGED); + + share->state.version=(ulong) time((time_t*) 0); + old_state= share->state; /* save state if not stored */ + r_locks= share->r_locks; + w_locks= share->w_locks; + old_lock= info->lock_type; + + /* Put same locks as old file */ + share->r_locks= share->w_locks= share->tot_locks= 0; + (void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE); + (void) mysql_file_close(share->kfile, MYF(MY_WME)); + share->kfile = -1; + (void) mysql_file_close(new_file, MYF(MY_WME)); + if (change_to_newfile(share->index_file_name,MI_NAME_IEXT,INDEX_TMP_EXT, + 0, MYF(0)) || + mi_open_keyfile(share)) + goto err2; + info->lock_type= F_UNLCK; /* Force mi_readinfo to lock */ + _mi_readinfo(info,F_WRLCK,0); /* Will lock the table */ + info->lock_type= old_lock; + share->r_locks= r_locks; + share->w_locks= w_locks; + share->tot_locks= r_locks+w_locks; + share->state= old_state; /* Restore old state */ + + info->state->key_file_length=param->new_file_pos; + info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + for (key=0 ; key < info->s->base.keys ; key++) + info->s->state.key_root[key]=index_pos[key]; + for (key=0 ; key < info->s->state.header.max_block_size_index ; key++) + info->s->state.key_del[key]= HA_OFFSET_ERROR; + + info->s->state.changed&= ~STATE_NOT_SORTED_PAGES; + DBUG_RETURN(0); + +err: + (void) mysql_file_close(new_file, MYF(MY_WME)); +err2: + (void) mysql_file_delete(mi_key_file_datatmp, + param->temp_filename, MYF(MY_WME)); + DBUG_RETURN(-1); +} /* mi_sort_index */ + + + /* Sort records recursive using one index */ + +static int sort_one_index(HA_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo, + my_off_t pagepos, File new_file) +{ + uint length,nod_flag,used_length, key_length; + uchar *buff,*keypos,*endpos; + uchar key[HA_MAX_POSSIBLE_KEY_BUFF]; + my_off_t new_page_pos,next_page; + char llbuff[22]; + DBUG_ENTER("sort_one_index"); + + /* cannot walk over R-tree indices */ + DBUG_ASSERT(keyinfo->key_alg != HA_KEY_ALG_RTREE); + new_page_pos=param->new_file_pos; + param->new_file_pos+=keyinfo->block_length; + + if (!(buff=(uchar*) my_alloca((uint) keyinfo->block_length))) + { + mi_check_print_error(param,"Not enough memory for key block"); + DBUG_RETURN(-1); + } + if (!_mi_fetch_keypage(info,keyinfo,pagepos,DFLT_INIT_HITS,buff,0)) + { + mi_check_print_error(param,"Can't read key block from filepos: %s", + llstr(pagepos,llbuff)); + goto err; + } + if ((nod_flag=mi_test_if_nod(buff)) || keyinfo->flag & HA_FULLTEXT) + { + used_length=mi_getint(buff); + keypos=buff+2+nod_flag; + endpos=buff+used_length; + for ( ;; ) + { + if (nod_flag) + { + next_page=_mi_kpos(nod_flag,keypos); + _mi_kpointer(info,keypos-nod_flag,param->new_file_pos); /* Save new pos */ + if (sort_one_index(param,info,keyinfo,next_page,new_file)) + { + DBUG_PRINT("error", + ("From page: %ld, keyoffset: %lu used_length: %d", + (ulong) pagepos, (ulong) (keypos - buff), + (int) used_length)); + DBUG_DUMP("buff",(uchar*) buff,used_length); + goto err; + } + } + if (keypos >= endpos || + (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0) + break; + DBUG_ASSERT(keypos <= endpos); + if (keyinfo->flag & HA_FULLTEXT) + { + uint off; + int subkeys; + get_key_full_length_rdonly(off, key); + subkeys=ft_sintXkorr(key+off); + if (subkeys < 0) + { + next_page= _mi_dpos(info,0,key+key_length); + _mi_dpointer(info,keypos-nod_flag-info->s->rec_reflength, + param->new_file_pos); /* Save new pos */ + if (sort_one_index(param,info,&info->s->ft2_keyinfo, + next_page,new_file)) + goto err; + } + } + } + } + + /* Fill block with zero and write it to the new index file */ + length=mi_getint(buff); + bzero((uchar*) buff+length,keyinfo->block_length-length); + if (mysql_file_pwrite(new_file, (uchar*) buff, (uint) keyinfo->block_length, + new_page_pos, MYF(MY_NABP | MY_WAIT_IF_FULL))) + { + mi_check_print_error(param,"Can't write indexblock, error: %d",my_errno); + goto err; + } + my_afree((uchar*) buff); + DBUG_RETURN(0); +err: + my_afree((uchar*) buff); + DBUG_RETURN(1); +} /* sort_one_index */ + + + /* + Let temporary file replace old file. + This assumes that the new file was created in the same + directory as given by realpath(filename). + This will ensure that any symlinks that are used will still work. + Copy stats from old file to new file, deletes orignal and + changes new file name to old file name + */ + +int change_to_newfile(const char * filename, const char * old_ext, + const char * new_ext, + time_t backup_time, + myf MyFlags) +{ + char old_filename[FN_REFLEN],new_filename[FN_REFLEN]; + /* Get real path to filename */ + (void) fn_format(old_filename,filename,"",old_ext,2+4+32); + return my_redel(old_filename, + fn_format(new_filename,old_filename,"",new_ext,2+4), + backup_time, MYF(MY_WME | MY_LINK_WARNING | MyFlags)); +} /* change_to_newfile */ + + + /* Locks a whole file */ + /* Gives an error-message if file can't be locked */ + +int lock_file(HA_CHECK *param, File file, my_off_t start, int lock_type, + const char *filetype, const char *filename) +{ + if (my_lock(file,lock_type,start,F_TO_EOF, + param->testflag & T_WAIT_FOREVER ? MYF(MY_SEEK_NOT_DONE) : + MYF(MY_SEEK_NOT_DONE | MY_SHORT_WAIT))) + { + mi_check_print_error(param," %d when locking %s '%s'",my_errno,filetype,filename); + param->error_printed=2; /* Don't give that data is crashed */ + return 1; + } + return 0; +} /* lock_file */ + + + /* Copy a block between two files */ + +int filecopy(HA_CHECK *param, File to,File from,my_off_t start, + my_off_t length, const char *type) +{ + char tmp_buff[IO_SIZE],*buff; + ulong buff_length; + DBUG_ENTER("filecopy"); + + buff_length=(ulong) MY_MIN(param->write_buffer_length,length); + if (!(buff=my_malloc(mi_key_memory_filecopy, buff_length, MYF(0)))) + { + buff=tmp_buff; buff_length=IO_SIZE; + } + + mysql_file_seek(from, start, MY_SEEK_SET, MYF(0)); + while (length > buff_length) + { + if (mysql_file_read(from, (uchar*) buff, buff_length, MYF(MY_NABP)) || + mysql_file_write(to, (uchar*) buff, buff_length, param->myf_rw)) + goto err; + length-= buff_length; + } + if (mysql_file_read(from, (uchar*) buff, (uint) length, MYF(MY_NABP)) || + mysql_file_write(to, (uchar*) buff, (uint) length, param->myf_rw)) + goto err; + if (buff != tmp_buff) + my_free(buff); + DBUG_RETURN(0); +err: + if (buff != tmp_buff) + my_free(buff); + mi_check_print_error(param,"Can't copy %s to tempfile, error %d", + type,my_errno); + DBUG_RETURN(1); +} + + +/* + Repair table or given index using sorting + + SYNOPSIS + mi_repair_by_sort() + param Repair parameters + info MyISAM handler to repair + name Name of table (for warnings) + rep_quick set to <> 0 if we should not change data file + + RESULT + 0 ok + <>0 Error +*/ + +int mi_repair_by_sort(HA_CHECK *param, register MI_INFO *info, + const char * name, int rep_quick) +{ + int got_error; + uint i; + ulong length; + ha_rows start_records; + my_off_t new_header_length,del; + File new_file; + MI_SORT_PARAM sort_param; + MYISAM_SHARE *share=info->s; + HA_KEYSEG *keyseg; + ulong *rec_per_key_part; + char llbuff[22], llbuff2[22]; + MI_SORT_INFO sort_info; + ulonglong UNINIT_VAR(key_map); + DBUG_ENTER("mi_repair_by_sort"); + + start_records=info->state->records; + got_error=1; + new_file= -1; + new_header_length=(param->testflag & T_UNPACK) ? 0 : + share->pack.header_length; + if (!(param->testflag & T_SILENT)) + { + printf("- recovering (with sort) MyISAM-table '%s'\n",name); + printf("Data records: %s\n", llstr(start_records,llbuff)); + } + param->testflag|=T_REP_BY_SORT; /* for easy checking */ + param->retry_repair= 0; + param->warning_printed= param->note_printed= 0; + param->error_printed= 0; + + if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD)) + param->testflag|=T_CALC_CHECKSUM; + + bzero((char*)&sort_info,sizeof(sort_info)); + bzero((char *)&sort_param, sizeof(sort_param)); + + if (!(sort_info.key_block= + alloc_key_blocks(param, + (uint) param->sort_key_blocks, + share->base.max_key_block_length))) + goto err; + + if (init_io_cache(¶m->read_cache,info->dfile, + (uint) param->read_buffer_length, + READ_CACHE,share->pack.header_length,1,MYF(MY_WME))) + goto err; + + sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks; + info->opt_flag|=WRITE_CACHE_USED; + + if (!mi_alloc_rec_buff(info, -1, &sort_param.record) || + !mi_alloc_rec_buff(info, -1, &sort_param.rec_buff)) + { + mi_check_print_error(param, "Not enough memory for extra record"); + goto err; + } + if (!rep_quick) + { + /* Get real path for data file */ + if ((new_file= mysql_file_create(mi_key_file_datatmp, + fn_format(param->temp_filename, + share->data_file_name, "", + DATA_TMP_EXT, 2+4), + 0, param->tmpfile_createflag, + MYF(0))) < 0) + { + mi_check_print_error(param,"Can't create new tempfile: '%s'", + param->temp_filename); + goto err; + } + if (new_header_length && + filecopy(param, new_file,info->dfile,0L,new_header_length, + "datafile-header")) + goto err; + if (param->testflag & T_UNPACK) + { + share->options&= ~HA_OPTION_COMPRESS_RECORD; + mi_int2store(share->state.header.options,share->options); + } + share->state.dellink= HA_OFFSET_ERROR; + if (init_io_cache(&info->rec_cache, new_file, + (uint) param->write_buffer_length, + WRITE_CACHE, new_header_length, 1, + MYF((param->myf_rw & MY_WAIT_IF_FULL) | MY_WME))) + goto err; + } + + info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + + /* Optionally drop indexes and optionally modify the key_map. */ + mi_drop_all_indexes(param, info, FALSE); + key_map= share->state.key_map; + if (param->testflag & T_CREATE_MISSING_KEYS) + { + /* Invert the copied key_map to recreate all disabled indexes. */ + key_map= ~key_map; + } + + sort_info.info=info; + sort_info.param = param; + + set_data_file_type(&sort_info, share); + sort_param.filepos=new_header_length; + sort_info.dupp=0; + sort_info.buff=0; + param->read_cache.end_of_file=sort_info.filelength= + mysql_file_seek(param->read_cache.file, 0L, MY_SEEK_END, MYF(0)); + if (info->state->data_file_length == 0) + info->state->data_file_length= sort_info.filelength; + + sort_param.wordlist=NULL; + init_alloc_root(mi_key_memory_MI_SORT_PARAM_wordroot, &sort_param.wordroot, + FTPARSER_MEMROOT_ALLOC_SIZE, 0, MYF(param->malloc_flags)); + + if (share->data_file_type == DYNAMIC_RECORD) + length=MY_MAX(share->base.min_pack_length+1,share->base.min_block_length); + else if (share->data_file_type == COMPRESSED_RECORD) + length=share->base.min_block_length; + else + length=share->base.pack_reclength; + sort_info.max_records= + ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records : + (ha_rows) (sort_info.filelength/length+1)); + sort_param.key_cmp=sort_key_cmp; + sort_param.lock_in_memory=lock_memory; + sort_param.tmpdir=param->tmpdir; + sort_param.sort_info=&sort_info; + sort_param.fix_datafile= (my_bool) (! rep_quick); + sort_param.master =1; + + del=info->state->del; + param->glob_crc=0; + if (param->testflag & T_CALC_CHECKSUM) + sort_param.calc_checksum= 1; + + rec_per_key_part= param->rec_per_key_part; + for (sort_param.key=0 ; sort_param.key < share->base.keys ; + rec_per_key_part+=sort_param.keyinfo->keysegs, sort_param.key++) + { + sort_param.read_cache=param->read_cache; + sort_param.keyinfo=share->keyinfo+sort_param.key; + sort_param.seg=sort_param.keyinfo->seg; + /* + Skip this index if it is marked disabled in the copied + (and possibly inverted) key_map. + */ + if (! mi_is_key_active(key_map, sort_param.key)) + { + /* Remember old statistics for key */ + memcpy((char*) rec_per_key_part, + (char*) (share->state.rec_per_key_part + + (uint) (rec_per_key_part - param->rec_per_key_part)), + sort_param.keyinfo->keysegs*sizeof(*rec_per_key_part)); + DBUG_PRINT("repair", ("skipping seemingly disabled index #: %u", + sort_param.key)); + continue; + } + + if ((!(param->testflag & T_SILENT))) + printf ("- Fixing index %d\n",sort_param.key+1); + sort_param.max_pos=sort_param.pos=share->pack.header_length; + keyseg=sort_param.seg; + bzero((char*) sort_param.unique,sizeof(sort_param.unique)); + sort_param.key_length=share->rec_reflength; + for (i=0 ; keyseg[i].type != HA_KEYTYPE_END; i++) + { + sort_param.key_length+=keyseg[i].length; + if (keyseg[i].flag & HA_SPACE_PACK) + sort_param.key_length+=get_pack_length(keyseg[i].length); + if (keyseg[i].flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART)) + sort_param.key_length+= 2 + MY_TEST(keyseg[i].length >= 127); + if (keyseg[i].flag & HA_NULL_PART) + sort_param.key_length++; + } + info->state->records=info->state->del=share->state.split=0; + info->state->empty=0; + + if (sort_param.keyinfo->flag & HA_FULLTEXT) + { + uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT* + sort_param.keyinfo->seg->charset->mbmaxlen; + sort_param.key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN; + /* + fulltext indexes may have much more entries than the + number of rows in the table. We estimate the number here. + */ + if (sort_param.keyinfo->parser == &ft_default_parser) + { + /* + for built-in parser the number of generated index entries + cannot be larger than the size of the data file divided + by the minimal word's length + */ + sort_info.max_records= + (ha_rows) (sort_info.filelength/ft_min_word_len+1); + } + else + { + /* + for external plugin parser we cannot tell anything at all :( + so, we'll use all the sort memory and start from ~10 buffpeks. + (see _create_index_by_sort) + */ + sort_info.max_records= 10 * + MY_MAX(param->sort_buffer_length, MIN_SORT_BUFFER) / + sort_param.key_length; + } + + sort_param.key_read=sort_ft_key_read; + sort_param.key_write=sort_ft_key_write; + } + else + { + sort_param.key_read=sort_key_read; + sort_param.key_write=sort_key_write; + } + + if (_create_index_by_sort(&sort_param, + (my_bool) (!(param->testflag & T_VERBOSE)), + param->sort_buffer_length)) + { + if ((param->testflag & T_CREATE_UNIQUE_BY_SORT) && sort_param.sort_info->dupp) + share->state.dupp_key= sort_param.key; + else + param->retry_repair= 1; + if (! param->error_printed) + mi_check_print_error(param, "Couldn't fix table with create_index_by_sort(). Error: %d", + my_errno); + goto err; + } + /* No need to calculate checksum again. */ + sort_param.calc_checksum= 0; + free_root(&sort_param.wordroot, MYF(0)); + + /* Set for next loop */ + sort_info.max_records= (ha_rows) info->state->records; + + if (param->testflag & T_STATISTICS) + update_key_parts(sort_param.keyinfo, rec_per_key_part, sort_param.unique, + param->stats_method == MI_STATS_METHOD_IGNORE_NULLS? + sort_param.notnull: NULL, + (ulonglong) info->state->records); + /* Enable this index in the permanent (not the copied) key_map. */ + mi_set_key_active(share->state.key_map, sort_param.key); + DBUG_PRINT("repair", ("set enabled index #: %u", sort_param.key)); + + if (sort_param.fix_datafile) + { + param->read_cache.end_of_file=sort_param.filepos; + if (write_data_suffix(&sort_info,1) || end_io_cache(&info->rec_cache)) + goto err; + if (param->testflag & T_SAFE_REPAIR) + { + /* Don't repair if we loosed more than one row */ + if (info->state->records+1 < start_records) + { + mi_check_print_error(param, + "Couldn't fix table as SAFE_REPAIR was requested and we would loose too many rows. %s -> %s", + llstr(start_records, llbuff), llstr(info->state->records, llbuff2)); + info->state->records= start_records; + goto err; + } + } + share->state.state.data_file_length = info->state->data_file_length= + sort_param.filepos; + /* Only whole records */ + share->state.version=(ulong) time((time_t*) 0); + mysql_file_close(info->dfile, MYF(0)); + info->dfile=new_file; + share->data_file_type=sort_info.new_data_file_type; + share->pack.header_length=(ulong) new_header_length; + sort_param.fix_datafile=0; + } + else + info->state->data_file_length=sort_param.max_pos; + + param->read_cache.file=info->dfile; /* re-init read cache */ + reinit_io_cache(¶m->read_cache,READ_CACHE,share->pack.header_length, + 1,1); + } + + if (param->testflag & T_WRITE_LOOP) + { + (void) fputs(" \r",stdout); (void) fflush(stdout); + } + + if (rep_quick && del+sort_info.dupp != info->state->del) + { + mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records"); + mi_check_print_error(param,"Run recovery again without --quick"); + got_error=1; + param->retry_repair=1; + param->testflag|=T_RETRY_WITHOUT_QUICK; + goto err; + } + + if (rep_quick && (param->testflag & T_FORCE_UNIQUENESS)) + { + my_off_t skr=info->state->data_file_length+ + (share->options & HA_OPTION_COMPRESS_RECORD ? + MEMMAP_EXTRA_MARGIN : 0); +#ifdef USE_RELOC + if (share->data_file_type == STATIC_RECORD && + skr < share->base.reloc*share->base.min_pack_length) + skr=share->base.reloc*share->base.min_pack_length; +#endif + if (skr != sort_info.filelength) + if (mysql_file_chsize(info->dfile, skr, 0, MYF(0))) + mi_check_print_warning(param, + "Can't change size of datafile, error: %d", + my_errno); + } + if (param->testflag & T_CALC_CHECKSUM) + info->state->checksum=param->glob_crc; + + if (mysql_file_chsize(share->kfile, info->state->key_file_length, 0, MYF(0))) + mi_check_print_warning(param, + "Can't change size of indexfile, error: %d", + my_errno); + + if (!(param->testflag & T_SILENT)) + { + if (start_records != info->state->records) + printf("Data records: %s\n", llstr(info->state->records,llbuff)); + if (sort_info.dupp) + mi_check_print_warning(param, + "%s records have been removed", + llstr(sort_info.dupp,llbuff)); + } + got_error=0; + + if (&share->state.state != info->state) + memcpy( &share->state.state, info->state, sizeof(*info->state)); + +err: + got_error|= flush_blocks(param, share->key_cache, share->kfile, + &share->dirty_part_map); + (void) end_io_cache(&info->rec_cache); + if (!got_error) + { + /* Replace the actual file with the temporary file */ + if (new_file >= 0) + { + got_error= replace_data_file(param, info, new_file); + new_file= -1; + } + } + if (got_error) + { + if (! param->error_printed) + mi_check_print_error(param,"%d when fixing table",my_errno); + if (new_file >= 0) + { + (void) mysql_file_close(new_file, MYF(0)); + (void) mysql_file_delete(mi_key_file_datatmp, + param->temp_filename, MYF(MY_WME)); + if (info->dfile == new_file) /* Retry with key cache */ + if (unlikely(mi_open_datafile(info, share))) + param->retry_repair= 0; /* Safety */ + } + mi_mark_crashed_on_repair(info); + if (killed_ptr(param)) + param->retry_repair= 0; /* No use to retry repair */ + } + else if (key_map == share->state.key_map) + share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS; + share->state.changed|=STATE_NOT_SORTED_PAGES; + + my_free(mi_get_rec_buff_ptr(info, sort_param.rec_buff)); + my_free(mi_get_rec_buff_ptr(info, sort_param.record)); + my_free(sort_info.key_block); + my_free(sort_info.ft_buf); + my_free(sort_info.buff); + (void) end_io_cache(¶m->read_cache); + info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED); + if (!got_error && (param->testflag & T_UNPACK)) + { + share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD; + share->pack.header_length=0; + } + DBUG_RETURN(got_error); +} + +/* + Threaded repair of table using sorting + + SYNOPSIS + mi_repair_parallel() + param Repair parameters + info MyISAM handler to repair + name Name of table (for warnings) + rep_quick set to <> 0 if we should not change data file + + DESCRIPTION + Same as mi_repair_by_sort but do it multithreaded + Each key is handled by a separate thread. + TODO: make a number of threads a parameter + + In parallel repair we use one thread per index. There are two modes: + + Quick + + Only the indexes are rebuilt. All threads share a read buffer. + Every thread that needs fresh data in the buffer enters the shared + cache lock. The last thread joining the lock reads the buffer from + the data file and wakes all other threads. + + Non-quick + + The data file is rebuilt and all indexes are rebuilt to point to + the new record positions. One thread is the master thread. It + reads from the old data file and writes to the new data file. It + also creates one of the indexes. The other threads read from a + buffer which is filled by the master. If they need fresh data, + they enter the shared cache lock. If the masters write buffer is + full, it flushes it to the new data file and enters the shared + cache lock too. When all threads joined in the lock, the master + copies its write buffer to the read buffer for the other threads + and wakes them. + + RESULT + 0 ok + <>0 Error +*/ + +int mi_repair_parallel(HA_CHECK *param, register MI_INFO *info, + const char * name, int rep_quick) +{ + int got_error; + uint i,key, istep; + ulong rec_length; + ha_rows start_records; + my_off_t new_header_length,del; + File new_file; + MI_SORT_PARAM *sort_param=0; + MYISAM_SHARE *share=info->s; + ulong *rec_per_key_part; + HA_KEYSEG *keyseg; + char llbuff[22]; + IO_CACHE new_data_cache; /* For non-quick repair. */ + IO_CACHE_SHARE io_share; + MI_SORT_INFO sort_info; + ulonglong UNINIT_VAR(key_map); + pthread_attr_t thr_attr; + ulong max_pack_reclength; + int error; + DBUG_ENTER("mi_repair_parallel"); + + start_records=info->state->records; + got_error=1; + new_file= -1; + new_header_length=(param->testflag & T_UNPACK) ? 0 : + share->pack.header_length; + if (!(param->testflag & T_SILENT)) + { + printf("- parallel recovering (with sort) MyISAM-table '%s'\n",name); + printf("Data records: %s\n", llstr(start_records,llbuff)); + } + param->testflag|=T_REP_PARALLEL; /* for easy checking */ + param->retry_repair= 0; + param->warning_printed= 0; + param->error_printed= 0; + + if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD)) + param->testflag|=T_CALC_CHECKSUM; + + /* + Quick repair (not touching data file, rebuilding indexes): + { + Read cache is (HA_CHECK *param)->read_cache using info->dfile. + } + + Non-quick repair (rebuilding data file and indexes): + { + Master thread: + + Read cache is (HA_CHECK *param)->read_cache using info->dfile. + Write cache is (MI_INFO *info)->rec_cache using new_file. + + Slave threads: + + Read cache is new_data_cache synced to master rec_cache. + + The final assignment of the filedescriptor for rec_cache is done + after the cache creation. + + Don't check file size on new_data_cache, as the resulting file size + is not known yet. + + As rec_cache and new_data_cache are synced, write_buffer_length is + used for the read cache 'new_data_cache'. Both start at the same + position 'new_header_length'. + } + */ + DBUG_PRINT("info", ("is quick repair: %d", rep_quick)); + bzero((char*)&sort_info,sizeof(sort_info)); + if (!rep_quick) + my_b_clear(&new_data_cache); + /* Initialize pthread structures before goto err. */ + mysql_mutex_init(mi_key_mutex_MI_SORT_INFO_mutex, + &sort_info.mutex, MY_MUTEX_INIT_FAST); + mysql_cond_init(mi_key_cond_MI_SORT_INFO_cond, &sort_info.cond, 0); + mysql_mutex_init(mi_key_mutex_MI_CHECK_print_msg, + ¶m->print_msg_mutex, MY_MUTEX_INIT_FAST); + param->need_print_msg_lock= 1; + + if (!(sort_info.key_block= + alloc_key_blocks(param, (uint) param->sort_key_blocks, + share->base.max_key_block_length))) + goto err; + + if (init_io_cache(¶m->read_cache, info->dfile, + (uint) param->read_buffer_length, + READ_CACHE, share->pack.header_length, 1, MYF(MY_WME))) + goto err; + + sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks; + info->opt_flag|=WRITE_CACHE_USED; + + if (!rep_quick) + { + /* Get real path for data file */ + if ((new_file= mysql_file_create(mi_key_file_datatmp, + fn_format(param->temp_filename, + share->data_file_name, "", + DATA_TMP_EXT, 2+4), + 0, param->tmpfile_createflag, + MYF(0))) < 0) + { + mi_check_print_error(param,"Can't create new tempfile: '%s'", + param->temp_filename); + goto err; + } + if (new_header_length && + filecopy(param, new_file,info->dfile,0L,new_header_length, + "datafile-header")) + goto err; + if (param->testflag & T_UNPACK) + { + share->options&= ~HA_OPTION_COMPRESS_RECORD; + mi_int2store(share->state.header.options,share->options); + } + share->state.dellink= HA_OFFSET_ERROR; + + if (init_io_cache(&info->rec_cache, new_file, + (uint) param->write_buffer_length, + WRITE_CACHE, new_header_length, 1, + MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw)) + goto err; + + if (init_io_cache(&new_data_cache, -1, + (uint) param->write_buffer_length, + READ_CACHE, new_header_length, 1, + MYF(MY_WME | MY_DONT_CHECK_FILESIZE))) + goto err; + } + + info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + + /* Optionally drop indexes and optionally modify the key_map. */ + mi_drop_all_indexes(param, info, FALSE); + key_map= share->state.key_map; + if (param->testflag & T_CREATE_MISSING_KEYS) + { + /* Invert the copied key_map to recreate all disabled indexes. */ + key_map= ~key_map; + } + + sort_info.info=info; + sort_info.param = param; + + set_data_file_type(&sort_info, share); + sort_info.dupp=0; + sort_info.buff=0; + param->read_cache.end_of_file=sort_info.filelength= + mysql_file_seek(param->read_cache.file, 0L, MY_SEEK_END, MYF(0)); + if (info->state->data_file_length == 0) + info->state->data_file_length= sort_info.filelength; + + if (share->data_file_type == DYNAMIC_RECORD) + rec_length=MY_MAX(share->base.min_pack_length+1,share->base.min_block_length); + else if (share->data_file_type == COMPRESSED_RECORD) + rec_length=share->base.min_block_length; + else + rec_length=share->base.pack_reclength; + /* + +1 below is required hack for parallel repair mode. + The info->state->records value, that is compared later + to sort_info.max_records and cannot exceed it, is + increased in sort_key_write. In mi_repair_by_sort, sort_key_write + is called after sort_key_read, where the comparison is performed, + but in parallel mode master thread can call sort_key_write + before some other repair thread calls sort_key_read. + Furthermore I'm not even sure +1 would be enough. + May be sort_info.max_records shold be always set to max value in + parallel mode. + */ + sort_info.max_records= + ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records + 1: + (ha_rows) (sort_info.filelength/rec_length+1)); + + del=info->state->del; + param->glob_crc=0; + /* for compressed tables */ + max_pack_reclength= MY_MAX(share->base.pack_reclength, share->vreclength); + if (share->options & HA_OPTION_COMPRESS_RECORD) + set_if_bigger(max_pack_reclength, share->max_pack_length); + if (!(sort_param=(MI_SORT_PARAM *) + my_malloc(mi_key_memory_MI_SORT_PARAM, (uint) share->base.keys * + (sizeof(MI_SORT_PARAM) + max_pack_reclength), + MYF(MY_ZEROFILL)))) + { + mi_check_print_error(param,"Not enough memory for key!"); + goto err; + } +#ifdef USING_SECOND_APPROACH + uint total_key_length=0; +#endif + rec_per_key_part= param->rec_per_key_part; + info->state->records=info->state->del=share->state.split=0; + info->state->empty=0; + + for (i=key=0, istep=1 ; key < share->base.keys ; + rec_per_key_part+=sort_param[i].keyinfo->keysegs, i+=istep, key++) + { + sort_param[i].key=key; + sort_param[i].keyinfo=share->keyinfo+key; + sort_param[i].seg=sort_param[i].keyinfo->seg; + /* + Skip this index if it is marked disabled in the copied + (and possibly inverted) key_map. + */ + if (! mi_is_key_active(key_map, key)) + { + /* Remember old statistics for key */ + memcpy((char*) rec_per_key_part, + (char*) (share->state.rec_per_key_part+ + (uint) (rec_per_key_part - param->rec_per_key_part)), + sort_param[i].keyinfo->keysegs*sizeof(*rec_per_key_part)); + istep=0; + continue; + } + istep=1; + if ((!(param->testflag & T_SILENT))) + printf ("- Fixing index %d\n",key+1); + if (sort_param[i].keyinfo->flag & HA_FULLTEXT) + { + sort_param[i].key_read=sort_ft_key_read; + sort_param[i].key_write=sort_ft_key_write; + } + else + { + sort_param[i].key_read=sort_key_read; + sort_param[i].key_write=sort_key_write; + } + sort_param[i].key_cmp=sort_key_cmp; + sort_param[i].lock_in_memory=lock_memory; + sort_param[i].tmpdir=param->tmpdir; + sort_param[i].sort_info=&sort_info; + sort_param[i].master=0; + sort_param[i].fix_datafile=0; + sort_param[i].calc_checksum= 0; + + sort_param[i].filepos=new_header_length; + sort_param[i].max_pos=sort_param[i].pos=share->pack.header_length; + + sort_param[i].record= (((uchar *)(sort_param+share->base.keys))+ + (max_pack_reclength * i)); + if (!mi_alloc_rec_buff(info, -1, &sort_param[i].rec_buff)) + { + mi_check_print_error(param,"Not enough memory!"); + goto err; + } + + sort_param[i].key_length=share->rec_reflength; + for (keyseg=sort_param[i].seg; keyseg->type != HA_KEYTYPE_END; + keyseg++) + { + sort_param[i].key_length+=keyseg->length; + if (keyseg->flag & HA_SPACE_PACK) + sort_param[i].key_length+=get_pack_length(keyseg->length); + if (keyseg->flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART)) + sort_param[i].key_length+= 2 + MY_TEST(keyseg->length >= 127); + if (keyseg->flag & HA_NULL_PART) + sort_param[i].key_length++; + } +#ifdef USING_SECOND_APPROACH + total_key_length+=sort_param[i].key_length; +#endif + + if (sort_param[i].keyinfo->flag & HA_FULLTEXT) + { + uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT* + sort_param[i].keyinfo->seg->charset->mbmaxlen; + sort_param[i].key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN; + init_alloc_root(mi_key_memory_MI_SORT_PARAM_wordroot, + &sort_param[i].wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0, + MYF(param->malloc_flags)); + } + } + sort_info.total_keys=i; + sort_param[0].master= 1; + sort_param[0].fix_datafile= (my_bool)(! rep_quick); + sort_param[0].calc_checksum= MY_TEST(param->testflag & T_CALC_CHECKSUM); + + if (!ftparser_alloc_param(info)) + goto err; + + sort_info.got_error=0; + mysql_mutex_lock(&sort_info.mutex); + + /* + Initialize the I/O cache share for use with the read caches and, in + case of non-quick repair, the write cache. When all threads join on + the cache lock, the writer copies the write cache contents to the + read caches. + */ + if (i > 1) + { + if (rep_quick) + init_io_cache_share(¶m->read_cache, &io_share, NULL, i); + else + init_io_cache_share(&new_data_cache, &io_share, &info->rec_cache, i); + } + else + io_share.total_threads= 0; /* share not used */ + + (void) pthread_attr_init(&thr_attr); + (void) pthread_attr_setdetachstate(&thr_attr,PTHREAD_CREATE_DETACHED); + + for (i=0 ; i < sort_info.total_keys ; i++) + { + /* + Copy the properly initialized IO_CACHE structure so that every + thread has its own copy. In quick mode param->read_cache is shared + for use by all threads. In non-quick mode all threads but the + first copy the shared new_data_cache, which is synchronized to the + write cache of the first thread. The first thread copies + param->read_cache, which is not shared. + */ + sort_param[i].read_cache= ((rep_quick || !i) ? param->read_cache : + new_data_cache); + DBUG_PRINT("io_cache_share", ("thread: %u read_cache: %p", + i, &sort_param[i].read_cache)); + + /* + two approaches: the same amount of memory for each thread + or the memory for the same number of keys for each thread... + In the second one all the threads will fill their sort_buffers + (and call write_keys) at the same time, putting more stress on i/o. + */ + sort_param[i].sortbuff_size= +#ifndef USING_SECOND_APPROACH + param->sort_buffer_length/sort_info.total_keys; +#else + param->sort_buffer_length*sort_param[i].key_length/total_key_length; +#endif + set_if_bigger(sort_param[i].sortbuff_size, MIN_SORT_BUFFER); + + if ((error= mysql_thread_create(mi_key_thread_find_all_keys, + &sort_param[i].thr, &thr_attr, + thr_find_all_keys, + (void *) (sort_param+i)))) + { + mi_check_print_error(param,"Cannot start a repair thread (errno= %d)", + error); + /* Cleanup: Detach from the share. Avoid others to be blocked. */ + if (io_share.total_threads) + remove_io_thread(&sort_param[i].read_cache); + DBUG_PRINT("error", ("Cannot start a repair thread")); + sort_info.got_error=1; + } + else + sort_info.threads_running++; + } + (void) pthread_attr_destroy(&thr_attr); + + /* waiting for all threads to finish */ + while (sort_info.threads_running) + mysql_cond_wait(&sort_info.cond, &sort_info.mutex); + mysql_mutex_unlock(&sort_info.mutex); + + if ((got_error= thr_write_keys(sort_param))) + { + param->retry_repair=1; + goto err; + } + got_error=1; /* Assume the following may go wrong */ + + if (sort_param[0].fix_datafile) + { + /* + Append some nuls to the end of a memory mapped file. Destroy the + write cache. The master thread did already detach from the share + by remove_io_thread() in sort.c:thr_find_all_keys(). + */ + if (write_data_suffix(&sort_info,1) || end_io_cache(&info->rec_cache)) + goto err; + if (param->testflag & T_SAFE_REPAIR) + { + /* Don't repair if we loosed more than one row */ + if (info->state->records+1 < start_records) + { + info->state->records=start_records; + goto err; + } + } + share->state.state.data_file_length= info->state->data_file_length= + sort_param->filepos; + /* Only whole records */ + share->state.version=(ulong) time((time_t*) 0); + + /* + Exchange the data file descriptor of the table, so that we use the + new file from now on. + */ + mysql_file_close(info->dfile, MYF(0)); + info->dfile=new_file; + + share->data_file_type=sort_info.new_data_file_type; + share->pack.header_length=(ulong) new_header_length; + } + else + info->state->data_file_length=sort_param->max_pos; + + if (rep_quick && del+sort_info.dupp != info->state->del) + { + mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records"); + mi_check_print_error(param,"Run recovery again without -q"); + param->retry_repair=1; + param->testflag|=T_RETRY_WITHOUT_QUICK; + goto err; + } + + if (rep_quick && (param->testflag & T_FORCE_UNIQUENESS)) + { + my_off_t skr=info->state->data_file_length+ + (share->options & HA_OPTION_COMPRESS_RECORD ? + MEMMAP_EXTRA_MARGIN : 0); +#ifdef USE_RELOC + if (share->data_file_type == STATIC_RECORD && + skr < share->base.reloc*share->base.min_pack_length) + skr=share->base.reloc*share->base.min_pack_length; +#endif + if (skr != sort_info.filelength) + if (mysql_file_chsize(info->dfile, skr, 0, MYF(0))) + mi_check_print_warning(param, + "Can't change size of datafile, error: %d", + my_errno); + } + if (param->testflag & T_CALC_CHECKSUM) + info->state->checksum=param->glob_crc; + + if (mysql_file_chsize(share->kfile, info->state->key_file_length, 0, MYF(0))) + mi_check_print_warning(param, + "Can't change size of indexfile, error: %d", my_errno); + + if (!(param->testflag & T_SILENT)) + { + if (start_records != info->state->records) + printf("Data records: %s\n", llstr(info->state->records,llbuff)); + if (sort_info.dupp) + mi_check_print_warning(param, + "%s records have been removed", + llstr(sort_info.dupp,llbuff)); + } + got_error=0; + + if (&share->state.state != info->state) + memcpy(&share->state.state, info->state, sizeof(*info->state)); + +err: + got_error|= flush_blocks(param, share->key_cache, share->kfile, + &share->dirty_part_map); + /* + Destroy the write cache. The master thread did already detach from + the share by remove_io_thread() or it was not yet started (if the + error happened before creating the thread). + */ + (void) end_io_cache(&info->rec_cache); + /* + Destroy the new data cache in case of non-quick repair. All slave + threads did either detach from the share by remove_io_thread() + already or they were not yet started (if the error happened before + creating the threads). + */ + if (!rep_quick && my_b_inited(&new_data_cache)) + (void) end_io_cache(&new_data_cache); + if (!got_error) + { + /* Replace the actual file with the temporary file */ + if (new_file >= 0) + { + got_error= replace_data_file(param, info, new_file); + new_file= -1; + } + } + if (got_error) + { + if (! param->error_printed) + mi_check_print_error(param,"%d when fixing table",my_errno); + if (new_file >= 0) + { + (void) mysql_file_close(new_file, MYF(0)); + (void) mysql_file_delete(mi_key_file_datatmp, + param->temp_filename, MYF(MY_WME)); + if (info->dfile == new_file) /* Retry with key cache */ + if (unlikely(mi_open_datafile(info, share))) + param->retry_repair= 0; /* Safety */ + } + mi_mark_crashed_on_repair(info); + if (killed_ptr(param)) + param->retry_repair= 0; + } + else if (key_map == share->state.key_map) + share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS; + share->state.changed|=STATE_NOT_SORTED_PAGES; + + mysql_cond_destroy(&sort_info.cond); + mysql_mutex_destroy(&sort_info.mutex); + mysql_mutex_destroy(¶m->print_msg_mutex); + param->need_print_msg_lock= 0; + + my_free(sort_info.ft_buf); + my_free(sort_info.key_block); + my_free(sort_param); + my_free(sort_info.buff); + (void) end_io_cache(¶m->read_cache); + info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED); + if (!got_error && (param->testflag & T_UNPACK)) + { + share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD; + share->pack.header_length=0; + } + DBUG_RETURN(got_error); +} + + /* Read next record and return next key */ + +static int sort_key_read(MI_SORT_PARAM *sort_param, void *key) +{ + int error; + MI_SORT_INFO *sort_info=sort_param->sort_info; + MI_INFO *info=sort_info->info; + DBUG_ENTER("sort_key_read"); + + if ((error=sort_get_next_record(sort_param))) + { + DBUG_ASSERT(error < 0 || + sort_info->param->error_printed || + sort_info->param->warning_printed || + sort_info->param->note_printed); + DBUG_RETURN(error); + } + if (info->state->records == sort_info->max_records) + { + my_errno= HA_ERR_WRONG_IN_RECORD; + mi_check_print_error(sort_info->param, + "Key %d - Found too many records; Can't continue", + sort_param->key+1); + DBUG_RETURN(1); + } + sort_param->real_key_length= + (info->s->rec_reflength+ + _mi_make_key(info, sort_param->key, (uchar*) key, + sort_param->record, sort_param->filepos)); +#ifdef HAVE_valgrind + bzero(key+sort_param->real_key_length, + (sort_param->key_length-sort_param->real_key_length)); +#endif + DBUG_RETURN(sort_write_record(sort_param)); +} /* sort_key_read */ + +static int sort_ft_key_read(MI_SORT_PARAM *sort_param, void *key) +{ + int error; + MI_SORT_INFO *sort_info=sort_param->sort_info; + MI_INFO *info=sort_info->info; + FT_WORD *wptr=0; + DBUG_ENTER("sort_ft_key_read"); + + if (!sort_param->wordlist) + { + for (;;) + { + free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE)); + if ((error=sort_get_next_record(sort_param))) + DBUG_RETURN(error); + if (!(wptr=_mi_ft_parserecord(info,sort_param->key,sort_param->record, + &sort_param->wordroot))) + DBUG_RETURN(1); + if (wptr->pos) + break; + error=sort_write_record(sort_param); + } + sort_param->wordptr=sort_param->wordlist=wptr; + } + else + { + error=0; + wptr=(FT_WORD*)(sort_param->wordptr); + } + + sort_param->real_key_length=(info->s->rec_reflength+ + _ft_make_key(info, sort_param->key, + key, wptr++, sort_param->filepos)); +#ifdef HAVE_valgrind + if (sort_param->key_length > sort_param->real_key_length) + bzero(key+sort_param->real_key_length, + (sort_param->key_length-sort_param->real_key_length)); +#endif + if (!wptr->pos) + { + free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE)); + sort_param->wordlist=0; + error=sort_write_record(sort_param); + } + else + sort_param->wordptr=(void*)wptr; + + DBUG_RETURN(error); +} /* sort_ft_key_read */ + + +/* + Read next record from file using parameters in sort_info. + + SYNOPSIS + sort_get_next_record() + sort_param Information about and for the sort process + + NOTE + + Dynamic Records With Non-Quick Parallel Repair + + For non-quick parallel repair we use a synchronized read/write + cache. This means that one thread is the master who fixes the data + file by reading each record from the old data file and writing it + to the new data file. By doing this the records in the new data + file are written contiguously. Whenever the write buffer is full, + it is copied to the read buffer. The slaves read from the read + buffer, which is not associated with a file. Thus read_cache.file + is -1. When using _mi_read_cache(), the slaves must always set + flag to READING_NEXT so that the function never tries to read from + file. This is safe because the records are contiguous. There is no + need to read outside the cache. This condition is evaluated in the + variable 'parallel_flag' for quick reference. read_cache.file must + be >= 0 in every other case. + + RETURN + -1 end of file + 0 ok + > 0 error +*/ + +static int sort_get_next_record(MI_SORT_PARAM *sort_param) +{ + int searching; + int parallel_flag; + uint found_record,b_type,left_length; + my_off_t pos; + uchar *UNINIT_VAR(to); + MI_BLOCK_INFO block_info; + MI_SORT_INFO *sort_info=sort_param->sort_info; + HA_CHECK *param=sort_info->param; + MI_INFO *info=sort_info->info; + MYISAM_SHARE *share=info->s; + char llbuff[22],llbuff2[22]; + DBUG_ENTER("sort_get_next_record"); + + if (killed_ptr(param)) + { + mi_check_print_error(param, "Repair killed by user with cause: %d", + (int) killed_ptr(param)); + param->retry_repair= 0; + DBUG_RETURN(1); + } + + switch (share->data_file_type) { + case STATIC_RECORD: + for (;;) + { + if (my_b_read(&sort_param->read_cache,sort_param->record, + share->base.pack_reclength)) + { + if (sort_param->read_cache.error) + param->out_flag |= O_DATA_LOST; + param->retry_repair=1; + param->testflag|=T_RETRY_WITHOUT_QUICK; + DBUG_RETURN(-1); + } + sort_param->start_recpos=sort_param->pos; + if (!sort_param->fix_datafile) + { + sort_param->filepos=sort_param->pos; + if (sort_param->master) + share->state.split++; + } + sort_param->max_pos=(sort_param->pos+=share->base.pack_reclength); + if (*sort_param->record) + { + if (sort_param->calc_checksum) + info->checksum= (*info->s->calc_check_checksum)(info, sort_param->record); + goto finish; + } + if (!sort_param->fix_datafile && sort_param->master) + { + info->state->del++; + info->state->empty+=share->base.pack_reclength; + } + } + case DYNAMIC_RECORD: + pos=sort_param->pos; + searching=(sort_param->fix_datafile && (param->testflag & T_EXTEND)); + parallel_flag= (sort_param->read_cache.file < 0) ? READING_NEXT : 0; + for (;;) + { + found_record=block_info.second_read= 0; + left_length=1; + if (searching) + { + pos=MY_ALIGN(pos,MI_DYN_ALIGN_SIZE); + param->testflag|=T_RETRY_WITHOUT_QUICK; + sort_param->start_recpos=pos; + } + do + { + if (pos > sort_param->max_pos) + sort_param->max_pos=pos; + if (pos & (MI_DYN_ALIGN_SIZE-1)) + { + if ((param->testflag & T_VERBOSE) || searching == 0) + mi_check_print_info(param,"Wrong aligned block at %s", + llstr(pos,llbuff)); + if (searching) + goto try_next; + } + if (found_record && pos == param->search_after_block) + mi_check_print_info(param,"Block: %s used by record at %s", + llstr(param->search_after_block,llbuff), + llstr(sort_param->start_recpos,llbuff2)); + if (_mi_read_cache(&sort_param->read_cache, + (uchar*) block_info.header,pos, + MI_BLOCK_INFO_HEADER_LENGTH, + (! found_record ? READING_NEXT : 0) | + parallel_flag | READING_HEADER)) + { + if (found_record) + { + mi_check_print_info(param, + "Can't read whole record at %s (errno: %d)", + llstr(sort_param->start_recpos,llbuff),errno); + goto try_next; + } + DBUG_RETURN(-1); + } + if (searching && ! sort_param->fix_datafile) + { + mi_check_print_info(param, + "Datafile is corrupted; Restart repair with option to copy datafile"); + param->error_printed=1; + param->retry_repair=1; + param->testflag|=T_RETRY_WITHOUT_QUICK; + my_errno= HA_ERR_WRONG_IN_RECORD; + DBUG_RETURN(1); /* Something wrong with data */ + } + b_type=_mi_get_block_info(&block_info,-1,pos); + if ((b_type & (BLOCK_ERROR | BLOCK_FATAL_ERROR)) || + ((b_type & BLOCK_FIRST) && + (block_info.rec_len < (uint) share->base.min_pack_length || + block_info.rec_len > (uint) share->base.max_pack_length))) + { + uint i; + if (param->testflag & T_VERBOSE || searching == 0) + mi_check_print_info(param, + "Wrong bytesec: %3d-%3d-%3d at %10s; Skipped", + block_info.header[0],block_info.header[1], + block_info.header[2],llstr(pos,llbuff)); + if (found_record) + goto try_next; + block_info.second_read=0; + searching=1; + /* Search after block in read header string */ + for (i=MI_DYN_ALIGN_SIZE ; + i < MI_BLOCK_INFO_HEADER_LENGTH ; + i+= MI_DYN_ALIGN_SIZE) + if (block_info.header[i] >= 1 && + block_info.header[i] <= MI_MAX_DYN_HEADER_BYTE) + break; + pos+=(ulong) i; + sort_param->start_recpos=pos; + continue; + } + if (b_type & BLOCK_DELETED) + { + my_bool error=0; + if (block_info.block_len+ (uint) (block_info.filepos-pos) < + share->base.min_block_length) + { + if (!searching) + mi_check_print_info(param, + "Deleted block with impossible length %u at %s", + block_info.block_len,llstr(pos,llbuff)); + error=1; + } + else + { + if ((block_info.next_filepos != HA_OFFSET_ERROR && + block_info.next_filepos >= + info->state->data_file_length) || + (block_info.prev_filepos != HA_OFFSET_ERROR && + block_info.prev_filepos >= info->state->data_file_length)) + { + if (!searching) + mi_check_print_info(param, + "Delete link points outside datafile at %s", + llstr(pos,llbuff)); + error=1; + } + } + if (error) + { + DBUG_ASSERT(param->note_printed); + if (found_record) + goto try_next; + searching=1; + pos+= MI_DYN_ALIGN_SIZE; + sort_param->start_recpos=pos; + block_info.second_read=0; + continue; + } + } + else + { + if (block_info.block_len+ (uint) (block_info.filepos-pos) < + share->base.min_block_length || + block_info.block_len > (uint) share->base.max_pack_length+ + MI_SPLIT_LENGTH) + { + if (!searching) + mi_check_print_info(param, + "Found block with impossible length %u at %s; Skipped", + block_info.block_len+ (uint) (block_info.filepos-pos), + llstr(pos,llbuff)); + if (found_record) + goto try_next; + searching=1; + pos+= MI_DYN_ALIGN_SIZE; + sort_param->start_recpos=pos; + block_info.second_read=0; + continue; + } + } + if (b_type & (BLOCK_DELETED | BLOCK_SYNC_ERROR)) + { + if (!sort_param->fix_datafile && sort_param->master && + (b_type & BLOCK_DELETED)) + { + info->state->empty+=block_info.block_len; + info->state->del++; + share->state.split++; + } + if (found_record) + { + mi_check_print_info(param, + "Found row block followed by deleted block"); + goto try_next; + } + if (searching) + { + pos+=MI_DYN_ALIGN_SIZE; + sort_param->start_recpos=pos; + } + else + pos=block_info.filepos+block_info.block_len; + block_info.second_read=0; + continue; + } + + if (!sort_param->fix_datafile && sort_param->master) + share->state.split++; + if (! found_record++) + { + sort_param->find_length=left_length=block_info.rec_len; + sort_param->start_recpos=pos; + if (!sort_param->fix_datafile) + sort_param->filepos=sort_param->start_recpos; + if (sort_param->fix_datafile && (param->testflag & T_EXTEND)) + sort_param->pos=block_info.filepos+1; + else + sort_param->pos=block_info.filepos+block_info.block_len; + if (share->base.blobs) + { + if (!(to=mi_alloc_rec_buff(info,block_info.rec_len, + &(sort_param->rec_buff)))) + { + if (param->max_record_length >= block_info.rec_len) + { + mi_check_print_error(param,"Not enough memory for blob at %s (need %lu)", + llstr(sort_param->start_recpos,llbuff), + (ulong) block_info.rec_len); + DBUG_ASSERT(param->error_printed); + DBUG_RETURN(1); + } + else + { + mi_check_print_info(param,"Not enough memory for blob at %s (need %lu); Row skipped", + llstr(sort_param->start_recpos,llbuff), + (ulong) block_info.rec_len); + goto try_next; + } + } + } + else + to= sort_param->rec_buff; + } + if (left_length < block_info.data_len || ! block_info.data_len) + { + mi_check_print_info(param, + "Found block with too small length at %s; Skipped", + llstr(sort_param->start_recpos,llbuff)); + goto try_next; + } + if (block_info.filepos + block_info.data_len > + sort_param->read_cache.end_of_file) + { + mi_check_print_info(param, + "Found block that points outside data file at %s", + llstr(sort_param->start_recpos,llbuff)); + goto try_next; + } + /* + Copy information that is already read. Avoid accessing data + below the cache start. This could happen if the header + streched over the end of the previous buffer contents. + */ + { + uint header_len= (uint) (block_info.filepos - pos); + uint prefetch_len= (MI_BLOCK_INFO_HEADER_LENGTH - header_len); + + if (prefetch_len > block_info.data_len) + prefetch_len= block_info.data_len; + if (prefetch_len) + { + memcpy(to, block_info.header + header_len, prefetch_len); + block_info.filepos+= prefetch_len; + block_info.data_len-= prefetch_len; + left_length-= prefetch_len; + to+= prefetch_len; + } + } + if (block_info.data_len && + _mi_read_cache(&sort_param->read_cache,to,block_info.filepos, + block_info.data_len, + (found_record == 1 ? READING_NEXT : 0) | + parallel_flag)) + { + mi_check_print_info(param, + "Read error for block at: %s (error: %d); Skipped", + llstr(block_info.filepos,llbuff),my_errno); + goto try_next; + } + left_length-=block_info.data_len; + to+=block_info.data_len; + pos=block_info.next_filepos; + if (pos == HA_OFFSET_ERROR && left_length) + { + mi_check_print_info(param,"Wrong block with wrong total length starting at %s", + llstr(sort_param->start_recpos,llbuff)); + goto try_next; + } + if (pos + MI_BLOCK_INFO_HEADER_LENGTH > sort_param->read_cache.end_of_file) + { + mi_check_print_info(param,"Found link that points at %s (outside data file) at %s", + llstr(pos,llbuff2), + llstr(sort_param->start_recpos,llbuff)); + goto try_next; + } + } while (left_length); + + if (_mi_rec_unpack(info,sort_param->record,sort_param->rec_buff, + sort_param->find_length) != MY_FILE_ERROR) + { + if (sort_param->calc_checksum) + info->checksum= (*info->s->calc_check_checksum)(info, + sort_param->record); + if ((param->testflag & (T_EXTEND | T_REP_ANY)) || searching) + { + if (_mi_rec_check(info, sort_param->record, sort_param->rec_buff, + sort_param->find_length, + (param->testflag & T_QUICK) && + sort_param->calc_checksum && + MY_TEST(info->s->calc_checksum))) + { + mi_check_print_info(param,"Found wrong packed record at %s", + llstr(sort_param->start_recpos,llbuff)); + goto try_next; + } + } + goto finish; + } + if (!searching) + mi_check_print_info(param,"Key %d - Found wrong stored record at %s", + sort_param->key+1, + llstr(sort_param->start_recpos,llbuff)); + try_next: + DBUG_ASSERT(param->error_printed || param->note_printed); + pos=(sort_param->start_recpos+=MI_DYN_ALIGN_SIZE); + searching=1; + } + case COMPRESSED_RECORD: + for (searching=0 ;; searching=1, sort_param->pos++) + { + if (_mi_read_cache(&sort_param->read_cache,(uchar*) block_info.header, + sort_param->pos, + share->pack.ref_length,READING_NEXT)) + DBUG_RETURN(-1); + if (searching && ! sort_param->fix_datafile) + { + param->error_printed=1; + param->retry_repair=1; + param->testflag|=T_RETRY_WITHOUT_QUICK; + my_errno= HA_ERR_WRONG_IN_RECORD; + DBUG_RETURN(1); /* Something wrong with data */ + } + sort_param->start_recpos=sort_param->pos; + if (_mi_pack_get_block_info(info, &sort_param->bit_buff, &block_info, + &sort_param->rec_buff, -1, sort_param->pos)) + DBUG_RETURN(-1); + if (!block_info.rec_len && + sort_param->pos + MEMMAP_EXTRA_MARGIN == + sort_param->read_cache.end_of_file) + DBUG_RETURN(-1); + if (block_info.rec_len < (uint) share->min_pack_length || + block_info.rec_len > (uint) share->max_pack_length) + { + if (! searching) + mi_check_print_info(param,"Found block with wrong recordlength: %d at %s\n", + block_info.rec_len, + llstr(sort_param->pos,llbuff)); + continue; + } + if (_mi_read_cache(&sort_param->read_cache,(uchar*) sort_param->rec_buff, + block_info.filepos, block_info.rec_len, + READING_NEXT)) + { + if (! searching) + mi_check_print_info(param,"Couldn't read whole record from %s", + llstr(sort_param->pos,llbuff)); + continue; + } + sort_param->rec_buff[block_info.rec_len]= 0; /* Keep valgrind happy */ + if (_mi_pack_rec_unpack(info, &sort_param->bit_buff, sort_param->record, + sort_param->rec_buff, block_info.rec_len)) + { + if (! searching) + mi_check_print_info(param,"Found wrong record at %s", + llstr(sort_param->pos,llbuff)); + continue; + } + if (!sort_param->fix_datafile) + { + sort_param->filepos=sort_param->pos; + if (sort_param->master) + share->state.split++; + } + sort_param->max_pos=(sort_param->pos=block_info.filepos+ + block_info.rec_len); + info->packed_length=block_info.rec_len; + if (sort_param->calc_checksum) + info->checksum= (*info->s->calc_check_checksum)(info, sort_param->record); + goto finish; + } + default: + DBUG_ASSERT(0); /* Impossible */ + break; + } + DBUG_ASSERT(0); /* Impossible */ + DBUG_RETURN(1); /* Impossible */ +finish: + if (sort_param->calc_checksum) + param->glob_crc+= info->checksum; + if (param->fix_record) + param->fix_record(info, sort_param->record, + param->testflag & T_REP_BY_SORT ? (int)sort_param->key + : -1); + DBUG_RETURN(0); +} + + +/* + Write record to new file. + + SYNOPSIS + sort_write_record() + sort_param Sort parameters. + + NOTE + This is only called by a master thread if parallel repair is used. + + RETURN + 0 OK + 1 Error +*/ + +int sort_write_record(MI_SORT_PARAM *sort_param) +{ + int flag; + uint length; + ulong block_length,reclength; + uchar *from; + uchar block_buff[8]; + MI_SORT_INFO *sort_info=sort_param->sort_info; + HA_CHECK *param=sort_info->param; + MI_INFO *info=sort_info->info; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("sort_write_record"); + + if (sort_param->fix_datafile) + { + switch (sort_info->new_data_file_type) { + case STATIC_RECORD: + if (my_b_write(&info->rec_cache,sort_param->record, + share->base.pack_reclength)) + { + mi_check_print_error(param,"%d when writing to datafile",my_errno); + DBUG_RETURN(1); + } + sort_param->filepos+=share->base.pack_reclength; + info->s->state.split++; + break; + case DYNAMIC_RECORD: + if (! info->blobs) + from=sort_param->rec_buff; + else + { + /* must be sure that local buffer is big enough */ + reclength=info->s->base.pack_reclength+ + _mi_calc_total_blob_length(info,sort_param->record)+ + ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+ + MI_DYN_DELETE_BLOCK_HEADER; + if (sort_info->buff_length < reclength) + { + if (!(sort_info->buff=my_realloc(mi_key_memory_SORT_INFO_buffer, + sort_info->buff, (uint) reclength, + MYF(MY_FREE_ON_ERROR | MY_WME | + MY_ALLOW_ZERO_PTR)))) + DBUG_RETURN(1); + sort_info->buff_length=reclength; + } + from= sort_info->buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER); + } + /* We can use info->checksum here as only one thread calls this. */ + info->checksum= (*info->s->calc_check_checksum)(info,sort_param->record); + reclength=_mi_rec_pack(info,from,sort_param->record); + flag=0; + + do + { + block_length= reclength + 3 + MY_TEST(reclength >= (65520 - 3)); + if (block_length < share->base.min_block_length) + block_length=share->base.min_block_length; + info->update|=HA_STATE_WRITE_AT_END; + block_length=MY_ALIGN(block_length,MI_DYN_ALIGN_SIZE); + if (block_length > MI_MAX_BLOCK_LENGTH) + block_length=MI_MAX_BLOCK_LENGTH; + if (_mi_write_part_record(info,0L,block_length, + sort_param->filepos+block_length, + &from,&reclength,&flag)) + { + mi_check_print_error(param,"%d when writing to datafile",my_errno); + DBUG_RETURN(1); + } + sort_param->filepos+=block_length; + info->s->state.split++; + } while (reclength); + /* sort_info->param->glob_crc+=info->checksum; */ + break; + case COMPRESSED_RECORD: + reclength=info->packed_length; + length= save_pack_length((uint) share->pack.version, block_buff, + reclength); + if (info->s->base.blobs) + length+= save_pack_length((uint) share->pack.version, + block_buff + length, info->blob_length); + if (my_b_write(&info->rec_cache,block_buff,length) || + my_b_write(&info->rec_cache,(uchar*) sort_param->rec_buff,reclength)) + { + mi_check_print_error(param,"%d when writing to datafile",my_errno); + DBUG_RETURN(1); + } + /* sort_info->param->glob_crc+=info->checksum; */ + sort_param->filepos+=reclength+length; + info->s->state.split++; + break; + default: + DBUG_ASSERT(0); /* Impossible */ + break; + } + } + if (sort_param->master) + { + info->state->records++; + if ((param->testflag & T_WRITE_LOOP) && + (info->state->records % WRITE_COUNT) == 0) + { + char llbuff[22]; + printf("%s\r", llstr(info->state->records,llbuff)); + (void) fflush(stdout); + } + } + DBUG_RETURN(0); +} /* sort_write_record */ + + + /* Compare two keys from _create_index_by_sort */ + +static int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a, + const void *b) +{ + uint not_used[2]; + return (ha_key_cmp(sort_param->seg, *((uchar**) a), *((uchar**) b), + USE_WHOLE_KEY, SEARCH_SAME, not_used)); +} /* sort_key_cmp */ + + +static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a) +{ + uint diff_pos[2]; + char llbuff[22],llbuff2[22]; + MI_SORT_INFO *sort_info=sort_param->sort_info; + HA_CHECK *param= sort_info->param; + int cmp; + + if (sort_info->key_block->inited) + { + cmp=ha_key_cmp(sort_param->seg, (uchar*) sort_info->key_block->lastkey, + (uchar*) a, USE_WHOLE_KEY, + SEARCH_FIND | SEARCH_UPDATE | SEARCH_INSERT, + diff_pos); + if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL) + ha_key_cmp(sort_param->seg, (uchar*) sort_info->key_block->lastkey, + (uchar*) a, USE_WHOLE_KEY, + SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diff_pos); + else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS) + { + diff_pos[0]= mi_collect_stats_nonulls_next(sort_param->seg, + sort_param->notnull, + (uchar*) sort_info-> + key_block->lastkey, + (uchar*)a); + } + sort_param->unique[diff_pos[0]-1]++; + } + else + { + cmp= -1; + if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS) + mi_collect_stats_nonulls_first(sort_param->seg, sort_param->notnull, + (uchar*)a); + } + if ((sort_param->keyinfo->flag & HA_NOSAME) && cmp == 0) + { + sort_info->dupp++; + sort_info->info->lastpos=get_record_for_key(sort_info->info, + sort_param->keyinfo, + (uchar*) a); + if ((param->testflag & (T_CREATE_UNIQUE_BY_SORT | T_SUPPRESS_ERR_HANDLING)) + == T_CREATE_UNIQUE_BY_SORT) + param->testflag|= T_SUPPRESS_ERR_HANDLING; + mi_check_print_warning(param, + "Duplicate key for record at %10s against record at %10s", + llstr(sort_info->info->lastpos,llbuff), + llstr(get_record_for_key(sort_info->info, + sort_param->keyinfo, + (uchar*) sort_info-> + key_block->lastkey), + llbuff2)); + param->testflag|=T_RETRY_WITHOUT_QUICK; + if (sort_info->param->testflag & T_VERBOSE) + _mi_print_key(stdout,sort_param->seg,(uchar*) a, USE_WHOLE_KEY); + return (sort_delete_record(sort_param)); + } +#ifndef DBUG_OFF + if (cmp > 0) + { + mi_check_print_error(param, + "Internal error: Keys are not in order from sort"); + return(1); + } +#endif + return (sort_insert_key(sort_param,sort_info->key_block, + (uchar*) a, HA_OFFSET_ERROR)); +} /* sort_key_write */ + +int sort_ft_buf_flush(MI_SORT_PARAM *sort_param) +{ + MI_SORT_INFO *sort_info=sort_param->sort_info; + SORT_KEY_BLOCKS *key_block=sort_info->key_block; + MYISAM_SHARE *share=sort_info->info->s; + uint val_off, val_len; + int error; + SORT_FT_BUF *ft_buf=sort_info->ft_buf; + uchar *from, *to; + + val_len=share->ft2_keyinfo.keylength; + get_key_full_length_rdonly(val_off, ft_buf->lastkey); + to= (uchar*) ft_buf->lastkey+val_off; + + if (ft_buf->buf) + { + /* flushing first-level tree */ + error=sort_insert_key(sort_param,key_block, (uchar*) ft_buf->lastkey, + HA_OFFSET_ERROR); + for (from=to+val_len; + !error && from < (uchar*) ft_buf->buf; + from+= val_len) + { + memcpy(to, from, val_len); + error=sort_insert_key(sort_param,key_block, (uchar*) ft_buf->lastkey, + HA_OFFSET_ERROR); + } + return error; + } + /* flushing second-level tree keyblocks */ + error=flush_pending_blocks(sort_param); + /* updating lastkey with second-level tree info */ + ft_intXstore(ft_buf->lastkey+val_off, -ft_buf->count); + _mi_dpointer(sort_info->info, (uchar*) ft_buf->lastkey+val_off+HA_FT_WLEN, + share->state.key_root[sort_param->key]); + /* restoring first level tree data in sort_info/sort_param */ + sort_info->key_block=sort_info->key_block_end- sort_info->param->sort_key_blocks; + sort_param->keyinfo=share->keyinfo+sort_param->key; + share->state.key_root[sort_param->key]=HA_OFFSET_ERROR; + /* writing lastkey in first-level tree */ + return error ? error : + sort_insert_key(sort_param,sort_info->key_block, + (uchar*) ft_buf->lastkey,HA_OFFSET_ERROR); +} + +static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a) +{ + uint a_len, val_off, val_len, error; + uchar *p; + MI_SORT_INFO *sort_info=sort_param->sort_info; + SORT_FT_BUF *ft_buf=sort_info->ft_buf; + SORT_KEY_BLOCKS *key_block=sort_info->key_block; + + val_len= HA_FT_WLEN + sort_info->info->s->rec_reflength; + get_key_full_length_rdonly(a_len, (uchar *)a); + + if (!ft_buf) + { + /* + use two-level tree only if key_reflength fits in rec_reflength place + and row format is NOT static - for _mi_dpointer not to garble offsets + */ + if ((sort_info->info->s->base.key_reflength <= + sort_info->info->s->rec_reflength) && + (sort_info->info->s->options & + (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD))) + ft_buf=(SORT_FT_BUF *)my_malloc(mi_key_memory_SORT_FT_BUF, + sort_param->keyinfo->block_length + + sizeof(SORT_FT_BUF), MYF(MY_WME)); + + if (!ft_buf) + { + sort_param->key_write=sort_key_write; + return sort_key_write(sort_param, a); + } + sort_info->ft_buf=ft_buf; + goto word_init_ft_buf; /* no need to duplicate the code */ + } + get_key_full_length_rdonly(val_off, ft_buf->lastkey); + + if (ha_compare_word(sort_param->seg->charset, + ((uchar *)a) + 1, a_len - 1, + (uchar*) ft_buf->lastkey + 1, val_off - 1) == 0) + { + if (!ft_buf->buf) /* store in second-level tree */ + { + ft_buf->count++; + return sort_insert_key(sort_param,key_block, + ((uchar *)a)+a_len, HA_OFFSET_ERROR); + } + + /* storing the key in the buffer. */ + memcpy (ft_buf->buf, (char *)a+a_len, val_len); + ft_buf->buf+=val_len; + if (ft_buf->buf < ft_buf->end) + return 0; + + /* converting to two-level tree */ + p= (uchar*) ft_buf->lastkey+val_off; + + while (key_block->inited) + key_block++; + sort_info->key_block=key_block; + sort_param->keyinfo=& sort_info->info->s->ft2_keyinfo; + ft_buf->count=(int)((uchar*) ft_buf->buf - p)/val_len; + + /* flushing buffer to second-level tree */ + for (error=0; !error && p < (uchar*) ft_buf->buf; p+= val_len) + error=sort_insert_key(sort_param,key_block,p,HA_OFFSET_ERROR); + ft_buf->buf=0; + return error; + } + + /* flushing buffer */ + if ((error=sort_ft_buf_flush(sort_param))) + return error; + +word_init_ft_buf: + a_len+=val_len; + memcpy(ft_buf->lastkey, a, a_len); + ft_buf->buf=ft_buf->lastkey+a_len; + /* + 32 is just a safety margin here + (at least MY_MAX(val_len, sizeof(nod_flag)) should be there). + May be better performance could be achieved if we'd put + (sort_info->keyinfo->block_length-32)/XXX + instead. + TODO: benchmark the best value for XXX. + */ + ft_buf->end=ft_buf->lastkey+ (sort_param->keyinfo->block_length-32); + return 0; +} /* sort_ft_key_write */ + + + /* get pointer to record from a key */ + +static my_off_t get_record_for_key(MI_INFO *info, MI_KEYDEF *keyinfo, + uchar *key) +{ + return _mi_dpos(info,0,key+_mi_keylength(keyinfo,key)); +} /* get_record_for_key */ + + + /* Insert a key in sort-key-blocks */ + +static int sort_insert_key(MI_SORT_PARAM *sort_param, + register SORT_KEY_BLOCKS *key_block, uchar *key, + my_off_t prev_block) +{ + uint a_length,t_length,nod_flag; + my_off_t filepos,key_file_length; + uchar *anc_buff,*lastkey; + MI_KEY_PARAM s_temp; + MI_INFO *info; + MI_KEYDEF *keyinfo=sort_param->keyinfo; + MI_SORT_INFO *sort_info= sort_param->sort_info; + HA_CHECK *param=sort_info->param; + DBUG_ENTER("sort_insert_key"); + + anc_buff= (uchar*) key_block->buff; + info=sort_info->info; + lastkey= (uchar*) key_block->lastkey; + nod_flag= (key_block == sort_info->key_block ? 0 : + info->s->base.key_reflength); + + if (!key_block->inited) + { + key_block->inited=1; + if (key_block == sort_info->key_block_end) + { + mi_check_print_error(param,"To many key-block-levels; Try increasing sort_key_blocks"); + DBUG_RETURN(1); + } + a_length=2+nod_flag; + key_block->end_pos= anc_buff+2; + lastkey=0; /* No previous key in block */ + } + else + a_length=mi_getint(anc_buff); + + /* Save pointer to previous block */ + if (nod_flag) + _mi_kpointer(info,(uchar*) key_block->end_pos,prev_block); + + t_length=(*keyinfo->pack_key)(keyinfo,nod_flag, + (uchar*) 0,lastkey,lastkey,key, + &s_temp); + (*keyinfo->store_key)(keyinfo, (uchar*) key_block->end_pos+nod_flag,&s_temp); + a_length+=t_length; + mi_putint(anc_buff,a_length,nod_flag); + key_block->end_pos+=t_length; + if (a_length <= keyinfo->block_length) + { + (void) _mi_move_key(keyinfo,key_block->lastkey,key); + key_block->last_length=a_length-t_length; + DBUG_RETURN(0); + } + + /* Fill block with end-zero and write filled block */ + mi_putint(anc_buff,key_block->last_length,nod_flag); + bzero((uchar*) anc_buff+key_block->last_length, + keyinfo->block_length- key_block->last_length); + key_file_length=info->state->key_file_length; + if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR) + DBUG_RETURN(1); + + /* If we read the page from the key cache, we have to write it back to it */ + if (key_file_length == info->state->key_file_length) + { + if (_mi_write_keypage(info, keyinfo, filepos, DFLT_INIT_HITS, anc_buff)) + DBUG_RETURN(1); + } + else if (mysql_file_pwrite(info->s->kfile, (uchar*) anc_buff, + (uint) keyinfo->block_length, filepos, + param->myf_rw)) + DBUG_RETURN(1); + DBUG_DUMP("buff",(uchar*) anc_buff,mi_getint(anc_buff)); + + /* Write separator-key to block in next level */ + if (sort_insert_key(sort_param,key_block+1,(uchar*) key_block->lastkey, + filepos)) + DBUG_RETURN(1); + + /* clear old block and write new key in it */ + key_block->inited=0; + DBUG_RETURN(sort_insert_key(sort_param, key_block,key,prev_block)); +} /* sort_insert_key */ + + + /* Delete record when we found a duplicated key */ + +static int sort_delete_record(MI_SORT_PARAM *sort_param) +{ + uint i; + int old_file,error; + uchar *key; + MI_SORT_INFO *sort_info=sort_param->sort_info; + HA_CHECK *param=sort_info->param; + MI_INFO *info=sort_info->info; + DBUG_ENTER("sort_delete_record"); + + if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK) + { + mi_check_print_error(param, + "Quick-recover aborted; Run recovery without switch -q or with switch -qq"); + DBUG_RETURN(1); + } + if (info->s->options & HA_OPTION_COMPRESS_RECORD) + { + mi_check_print_error(param, + "Recover aborted; Can't run standard recovery on compressed tables with errors in data-file. Use switch 'myisamchk --safe-recover' to fix it\n",stderr);; + DBUG_RETURN(1); + } + + old_file=info->dfile; + info->dfile=info->rec_cache.file; + if (sort_info->current_key) + { + key=info->lastkey+info->s->base.max_key_length; + if ((error=(*info->s->read_rnd)(info,sort_param->record,info->lastpos,0)) && + error != HA_ERR_RECORD_DELETED) + { + mi_check_print_error(param,"Can't read record to be removed"); + info->dfile=old_file; + DBUG_RETURN(1); + } + + for (i=0 ; i < sort_info->current_key ; i++) + { + uint key_length=_mi_make_key(info,i,key,sort_param->record,info->lastpos); + if (_mi_ck_delete(info,i,key,key_length)) + { + mi_check_print_error(param,"Can't delete key %d from record to be removed",i+1); + info->dfile=old_file; + DBUG_RETURN(1); + } + } + if (sort_param->calc_checksum) + param->glob_crc-=(*info->s->calc_checksum)(info, sort_param->record); + } + error=flush_io_cache(&info->rec_cache) || (*info->s->delete_record)(info); + info->dfile=old_file; /* restore actual value */ + info->state->records--; + DBUG_RETURN(error); +} /* sort_delete_record */ + + /* Fix all pending blocks and flush everything to disk */ + +int flush_pending_blocks(MI_SORT_PARAM *sort_param) +{ + uint nod_flag,length; + my_off_t filepos,key_file_length; + SORT_KEY_BLOCKS *key_block; + MI_SORT_INFO *sort_info= sort_param->sort_info; + myf myf_rw=sort_info->param->myf_rw; + MI_INFO *info=sort_info->info; + MI_KEYDEF *keyinfo=sort_param->keyinfo; + DBUG_ENTER("flush_pending_blocks"); + + filepos= HA_OFFSET_ERROR; /* if empty file */ + nod_flag=0; + for (key_block=sort_info->key_block ; key_block->inited ; key_block++) + { + key_block->inited=0; + length=mi_getint(key_block->buff); + if (nod_flag) + _mi_kpointer(info,(uchar*) key_block->end_pos,filepos); + key_file_length=info->state->key_file_length; + bzero((uchar*) key_block->buff+length, keyinfo->block_length-length); + if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR) + DBUG_RETURN(1); + + /* If we read the page from the key cache, we have to write it back */ + if (key_file_length == info->state->key_file_length) + { + if (_mi_write_keypage(info, keyinfo, filepos, + DFLT_INIT_HITS, (uchar*) key_block->buff)) + DBUG_RETURN(1); + } + else if (mysql_file_pwrite(info->s->kfile, (uchar*) key_block->buff, + (uint) keyinfo->block_length, filepos, myf_rw)) + DBUG_RETURN(1); + DBUG_DUMP("buff",(uchar*) key_block->buff,length); + nod_flag=1; + } + info->s->state.key_root[sort_param->key]=filepos; /* Last is root for tree */ + DBUG_RETURN(0); +} /* flush_pending_blocks */ + + /* alloc space and pointers for key_blocks */ + +static SORT_KEY_BLOCKS *alloc_key_blocks(HA_CHECK *param, uint blocks, + uint buffer_length) +{ + reg1 uint i; + SORT_KEY_BLOCKS *block; + DBUG_ENTER("alloc_key_blocks"); + + if (!(block=(SORT_KEY_BLOCKS*) my_malloc(mi_key_memory_SORT_KEY_BLOCKS, + (sizeof(SORT_KEY_BLOCKS)+ + buffer_length+IO_SIZE)*blocks, + MYF(0)))) + { + mi_check_print_error(param,"Not enough memory for sort-key-blocks"); + return(0); + } + for (i=0 ; i < blocks ; i++) + { + block[i].inited=0; + block[i].buff=(uchar*) (block+blocks)+(buffer_length+IO_SIZE)*i; + } + DBUG_RETURN(block); +} /* alloc_key_blocks */ + + + /* Check if file is almost full */ + +int test_if_almost_full(MI_INFO *info) +{ + if (info->s->options & HA_OPTION_COMPRESS_RECORD) + return 0; + return mysql_file_seek(info->s->kfile, 0L, MY_SEEK_END, + MYF(MY_THREADSAFE)) / 10 * 9 > + (my_off_t) info->s->base.max_key_file_length || + mysql_file_seek(info->dfile, 0L, MY_SEEK_END, + MYF(0)) / 10 * 9 > + (my_off_t) info->s->base.max_data_file_length; +} + + /* Recreate table with bigger more alloced record-data */ + +int recreate_table(HA_CHECK *param, MI_INFO **org_info, char *filename) +{ + int error; + MI_INFO info; + MYISAM_SHARE share; + MI_KEYDEF *keyinfo,*key,*key_end; + HA_KEYSEG *keysegs,*keyseg; + MI_COLUMNDEF *recdef,*rec,*end; + MI_UNIQUEDEF *uniquedef,*u_ptr,*u_end; + MI_STATUS_INFO status_info; + uint unpack,key_parts; + ha_rows max_records; + ulonglong file_length,tmp_length; + MI_CREATE_INFO create_info; + DBUG_ENTER("recreate_table"); + + error=1; /* Default error */ + info= **org_info; + status_info= (*org_info)->state[0]; + info.state= &status_info; + share= *(*org_info)->s; + unpack= (share.options & HA_OPTION_COMPRESS_RECORD) && + (param->testflag & T_UNPACK); + if (!(keyinfo=(MI_KEYDEF*) my_alloca(sizeof(MI_KEYDEF)*share.base.keys))) + DBUG_RETURN(0); + memcpy((uchar*) keyinfo,(uchar*) share.keyinfo, + (size_t) (sizeof(MI_KEYDEF)*share.base.keys)); + + key_parts= share.base.all_key_parts; + if (!(keysegs=(HA_KEYSEG*) my_alloca(sizeof(HA_KEYSEG)* + (key_parts+share.base.keys)))) + { + my_afree((uchar*) keyinfo); + DBUG_RETURN(1); + } + if (!(recdef=(MI_COLUMNDEF*) + my_alloca(sizeof(MI_COLUMNDEF)*(share.base.fields+1)))) + { + my_afree((uchar*) keyinfo); + my_afree((uchar*) keysegs); + DBUG_RETURN(1); + } + if (!(uniquedef=(MI_UNIQUEDEF*) + my_alloca(sizeof(MI_UNIQUEDEF)*(share.state.header.uniques+1)))) + { + my_afree((uchar*) recdef); + my_afree((uchar*) keyinfo); + my_afree((uchar*) keysegs); + DBUG_RETURN(1); + } + + /* Copy the column definitions */ + memcpy((uchar*) recdef,(uchar*) share.rec, + (size_t) (sizeof(MI_COLUMNDEF)*(share.base.fields+1))); + for (rec=recdef,end=recdef+share.base.fields; rec != end ; rec++) + { + if (unpack && !(share.options & HA_OPTION_PACK_RECORD) && + rec->type != FIELD_BLOB && + rec->type != FIELD_VARCHAR && + rec->type != FIELD_CHECK) + rec->type=(int) FIELD_NORMAL; + } + + /* Change the new key to point at the saved key segments */ + memcpy((uchar*) keysegs,(uchar*) share.keyparts, + (size_t) (sizeof(HA_KEYSEG)*(key_parts+share.base.keys+ + share.state.header.uniques))); + keyseg=keysegs; + for (key=keyinfo,key_end=keyinfo+share.base.keys; key != key_end ; key++) + { + key->seg=keyseg; + for (; keyseg->type ; keyseg++) + { + if (param->language) + keyseg->language=param->language; /* change language */ + } + keyseg++; /* Skip end pointer */ + } + + /* Copy the unique definitions and change them to point at the new key + segments*/ + memcpy((uchar*) uniquedef,(uchar*) share.uniqueinfo, + (size_t) (sizeof(MI_UNIQUEDEF)*(share.state.header.uniques))); + for (u_ptr=uniquedef,u_end=uniquedef+share.state.header.uniques; + u_ptr != u_end ; u_ptr++) + { + u_ptr->seg=keyseg; + keyseg+=u_ptr->keysegs+1; + } + unpack= (share.options & HA_OPTION_COMPRESS_RECORD) && + (param->testflag & T_UNPACK); + share.options&= ~HA_OPTION_TEMP_COMPRESS_RECORD; + + file_length=(ulonglong) mysql_file_seek(info.dfile, 0L, MY_SEEK_END, MYF(0)); + tmp_length= file_length+file_length/10; + set_if_bigger(file_length,param->max_data_file_length); + set_if_bigger(file_length,tmp_length); + set_if_bigger(file_length,(ulonglong) share.base.max_data_file_length); + + if (share.options & HA_OPTION_COMPRESS_RECORD) + share.base.records= max_records= info.state->records; + else if (!(share.options & HA_OPTION_PACK_RECORD)) + max_records= (ha_rows) (file_length / share.base.pack_reclength); + else + max_records= 0; + + (void) mi_close(*org_info); + bzero((char*) &create_info,sizeof(create_info)); + create_info.max_rows= max_records; + create_info.reloc_rows=share.base.reloc; + create_info.old_options=(share.options | + (unpack ? HA_OPTION_TEMP_COMPRESS_RECORD : 0)); + + create_info.data_file_length=file_length; + create_info.auto_increment=share.state.auto_increment; + create_info.language = (param->language ? param->language : + share.state.header.language); + create_info.key_file_length= status_info.key_file_length; + /* + Allow for creating an auto_increment key. This has an effect only if + an auto_increment key exists in the original table. + */ + create_info.with_auto_increment= TRUE; + /* We don't have to handle symlinks here because we are using + HA_DONT_TOUCH_DATA */ + if (mi_create(filename, + share.base.keys - share.state.header.uniques, + keyinfo, share.base.fields, recdef, + share.state.header.uniques, uniquedef, + &create_info, + HA_DONT_TOUCH_DATA)) + { + mi_check_print_error(param,"Got error %d when trying to recreate indexfile",my_errno); + goto end; + } + *org_info=mi_open(filename,O_RDWR, + (param->testflag & T_WAIT_FOREVER) ? HA_OPEN_WAIT_IF_LOCKED : + (param->testflag & T_DESCRIPT) ? HA_OPEN_IGNORE_IF_LOCKED : + HA_OPEN_ABORT_IF_LOCKED); + if (!*org_info) + { + mi_check_print_error(param,"Got error %d when trying to open re-created indexfile", + my_errno); + goto end; + } + /* We are modifing */ + (*org_info)->s->options&= ~HA_OPTION_READ_ONLY_DATA; + (void) _mi_readinfo(*org_info,F_WRLCK,0); + (*org_info)->state->records=info.state->records; + if (share.state.create_time) + (*org_info)->s->state.create_time=share.state.create_time; + (*org_info)->s->state.unique=(*org_info)->this_unique= + share.state.unique; + (*org_info)->state->checksum=info.state->checksum; + (*org_info)->state->del=info.state->del; + (*org_info)->s->state.dellink=share.state.dellink; + (*org_info)->state->empty=info.state->empty; + (*org_info)->state->data_file_length=info.state->data_file_length; + if (update_state_info(param,*org_info,UPDATE_TIME | UPDATE_STAT | + UPDATE_OPEN_COUNT)) + goto end; + error=0; +end: + my_afree((uchar*) uniquedef); + my_afree((uchar*) keyinfo); + my_afree((uchar*) recdef); + my_afree((uchar*) keysegs); + DBUG_RETURN(error); +} + + + /* write suffix to data file if neaded */ + +int write_data_suffix(MI_SORT_INFO *sort_info, my_bool fix_datafile) +{ + MI_INFO *info=sort_info->info; + + if (info->s->options & HA_OPTION_COMPRESS_RECORD && fix_datafile) + { + uchar buff[MEMMAP_EXTRA_MARGIN]; + bzero(buff,sizeof(buff)); + if (my_b_write(&info->rec_cache,buff,sizeof(buff))) + { + mi_check_print_error(sort_info->param, + "%d when writing to datafile",my_errno); + return 1; + } + sort_info->param->read_cache.end_of_file+=sizeof(buff); + } + return 0; +} + + /* Update state and myisamchk_time of indexfile */ + +int update_state_info(HA_CHECK *param, MI_INFO *info,uint update) +{ + MYISAM_SHARE *share=info->s; + + if (update & UPDATE_OPEN_COUNT) + { + share->state.open_count=0; + share->global_changed=0; + } + if (update & UPDATE_STAT) + { + uint i, key_parts= mi_uint2korr(share->state.header.key_parts); + share->state.rec_per_key_rows=info->state->records; + share->state.changed&= ~STATE_NOT_ANALYZED; + if (info->state->records) + { + for (i=0; i<key_parts; i++) + { + if (!(share->state.rec_per_key_part[i]=param->rec_per_key_part[i])) + share->state.changed|= STATE_NOT_ANALYZED; + } + } + } + if (update & (UPDATE_STAT | UPDATE_SORT | UPDATE_TIME | UPDATE_AUTO_INC)) + { + if (update & UPDATE_TIME) + { + share->state.check_time= time((time_t*) 0); + if (!share->state.create_time) + share->state.create_time=share->state.check_time; + } + /* + When tables are locked we haven't synched the share state and the + real state for a while so we better do it here before synching + the share state to disk. Only when table is write locked is it + necessary to perform this synch. + */ + if (info->lock_type == F_WRLCK) + share->state.state= *info->state; + if (mi_state_info_write(share->kfile,&share->state,1+2)) + goto err; + share->changed=0; + } + { /* Force update of status */ + int error; + uint r_locks=share->r_locks,w_locks=share->w_locks; + share->r_locks= share->w_locks= share->tot_locks= 0; + + DBUG_EXECUTE_IF("simulate_incorrect_share_wlock_value", + DEBUG_SYNC_C("after_share_wlock_set_to_0");); + + error=_mi_writeinfo(info,WRITEINFO_NO_UNLOCK); + share->r_locks=r_locks; + share->w_locks=w_locks; + share->tot_locks=r_locks+w_locks; + if (!error) + return 0; + } +err: + mi_check_print_error(param,"%d when updating keyfile",my_errno); + return 1; +} + + /* + Update auto increment value for a table + When setting the 'repair_only' flag we only want to change the + old auto_increment value if its wrong (smaller than some given key). + The reason is that we shouldn't change the auto_increment value + for a table without good reason when only doing a repair; If the + user have inserted and deleted rows, the auto_increment value + may be bigger than the biggest current row and this is ok. + + If repair_only is not set, we will update the flag to the value in + param->auto_increment is bigger than the biggest key. + */ + +void update_auto_increment_key(HA_CHECK *param, MI_INFO *info, + my_bool repair_only) +{ + uchar *record= 0; + DBUG_ENTER("update_auto_increment_key"); + + if (!info->s->base.auto_key || + ! mi_is_key_active(info->s->state.key_map, info->s->base.auto_key - 1)) + { + if (!(param->testflag & T_VERY_SILENT)) + mi_check_print_info(param, + "Table: %s doesn't have an auto increment key\n", + param->isam_file_name); + DBUG_VOID_RETURN; + } + if (!(param->testflag & T_SILENT) && + !(param->testflag & T_REP_ANY)) + printf("Updating MyISAM file: %s\n", param->isam_file_name); + /* + We have to use an allocated buffer instead of info->rec_buff as + _mi_put_key_in_record() may use info->rec_buff + */ + if (!mi_alloc_rec_buff(info, -1, &record)) + { + mi_check_print_error(param,"Not enough memory for extra record"); + DBUG_VOID_RETURN; + } + + mi_extra(info,HA_EXTRA_KEYREAD,0); + if (mi_rlast(info, record, info->s->base.auto_key-1)) + { + if (my_errno != HA_ERR_END_OF_FILE) + { + mi_extra(info,HA_EXTRA_NO_KEYREAD,0); + my_free(mi_get_rec_buff_ptr(info, record)); + mi_check_print_error(param,"%d when reading last record",my_errno); + DBUG_VOID_RETURN; + } + if (!repair_only) + info->s->state.auto_increment=param->auto_increment_value; + } + else + { + ulonglong auto_increment= retrieve_auto_increment(info, record); + set_if_bigger(info->s->state.auto_increment,auto_increment); + if (!repair_only) + set_if_bigger(info->s->state.auto_increment, param->auto_increment_value); + } + mi_extra(info,HA_EXTRA_NO_KEYREAD,0); + my_free(mi_get_rec_buff_ptr(info, record)); + update_state_info(param, info, UPDATE_AUTO_INC); + DBUG_VOID_RETURN; +} + + +/* + Update statistics for each part of an index + + SYNOPSIS + update_key_parts() + keyinfo IN Index information (only key->keysegs used) + rec_per_key_part OUT Store statistics here + unique IN Array of (#distinct tuples) + notnull_tuples IN Array of (#tuples), or NULL + records Number of records in the table + + DESCRIPTION + This function is called produce index statistics values from unique and + notnull_tuples arrays after these arrays were produced with sequential + index scan (the scan is done in two places: chk_index() and + sort_key_write()). + + This function handles all 3 index statistics collection methods. + + Unique is an array: + unique[0]= (#different values of {keypart1}) - 1 + unique[1]= (#different values of {keypart1,keypart2} tuple)-unique[0]-1 + ... + + For MI_STATS_METHOD_IGNORE_NULLS method, notnull_tuples is an array too: + notnull_tuples[0]= (#of {keypart1} tuples such that keypart1 is not NULL) + notnull_tuples[1]= (#of {keypart1,keypart2} tuples such that all + keypart{i} are not NULL) + ... + For all other statistics collection methods notnull_tuples==NULL. + + Output is an array: + rec_per_key_part[k] = + = E(#records in the table such that keypart_1=c_1 AND ... AND + keypart_k=c_k for arbitrary constants c_1 ... c_k) + + = {assuming that values have uniform distribution and index contains all + tuples from the domain (or that {c_1, ..., c_k} tuple is chosen from + index tuples} + + = #tuples-in-the-index / #distinct-tuples-in-the-index. + + The #tuples-in-the-index and #distinct-tuples-in-the-index have different + meaning depending on which statistics collection method is used: + + MI_STATS_METHOD_* how are nulls compared? which tuples are counted? + NULLS_EQUAL NULL == NULL all tuples in table + NULLS_NOT_EQUAL NULL != NULL all tuples in table + IGNORE_NULLS n/a tuples that don't have NULLs +*/ + +void update_key_parts(MI_KEYDEF *keyinfo, ulong *rec_per_key_part, + ulonglong *unique, ulonglong *notnull, + ulonglong records) +{ + ulonglong count=0,tmp, unique_tuples; + ulonglong tuples= records; + uint parts; + for (parts=0 ; parts < keyinfo->keysegs ; parts++) + { + count+=unique[parts]; + unique_tuples= count + 1; + if (notnull) + { + tuples= notnull[parts]; + /* + #(unique_tuples not counting tuples with NULLs) = + #(unique_tuples counting tuples with NULLs as different) - + #(tuples with NULLs) + */ + unique_tuples -= (records - notnull[parts]); + } + + if (unique_tuples == 0) + tmp= 1; + else if (count == 0) + tmp= tuples; /* 1 unique tuple */ + else + tmp= (tuples + unique_tuples/2) / unique_tuples; + + /* + for some weird keys (e.g. FULLTEXT) tmp can be <1 here. + let's ensure it is not + */ + set_if_bigger(tmp,1); + /* Keys are stored as 32 byte int's; Ensure we don't get an overflow */ + if (tmp >= (ulonglong) ~(uint32) 0) + tmp=(ulonglong) ~(uint32) 0; + + *rec_per_key_part=(ulong) tmp; + rec_per_key_part++; + } +} + + +static ha_checksum mi_byte_checksum(const uchar *buf, uint length) +{ + ha_checksum crc; + const uchar *end=buf+length; + for (crc=0; buf != end; buf++) + crc=((crc << 1) + *((uchar*) buf)) + + MY_TEST(crc & (((ha_checksum) 1) << (8 * sizeof(ha_checksum) - 1))); + return crc; +} + +my_bool mi_too_big_key_for_sort(MI_KEYDEF *key, ha_rows rows) +{ + uint key_maxlength=key->maxlength; + if (key->flag & HA_FULLTEXT) + { + uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT* + key->seg->charset->mbmaxlen; + key_maxlength+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN; + } + return (key->flag & HA_SPATIAL) || + (key->flag & (HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY | HA_FULLTEXT) && + ((ulonglong) rows * key_maxlength > myisam_max_temp_length)); +} + +/* + Return TRUE if we can use repair by sorting + One can set the force argument to force to use sorting + even if the temporary file would be quite big! +*/ + +my_bool mi_test_if_sort_rep(MI_INFO *info, ha_rows rows, + ulonglong key_map, my_bool force) +{ + MYISAM_SHARE *share=info->s; + MI_KEYDEF *key=share->keyinfo; + uint i; + + /* + mi_repair_by_sort only works if we have at least one key. If we don't + have any keys, we should use the normal repair. + */ + if (! mi_is_any_key_active(key_map)) + return FALSE; /* Can't use sort */ + for (i=0 ; i < share->base.keys ; i++,key++) + { + if (!force && mi_too_big_key_for_sort(key,rows)) + return FALSE; + } + return TRUE; +} + + +static void +set_data_file_type(MI_SORT_INFO *sort_info, MYISAM_SHARE *share) +{ + if ((sort_info->new_data_file_type=share->data_file_type) == + COMPRESSED_RECORD && sort_info->param->testflag & T_UNPACK) + { + MYISAM_SHARE tmp; + + if (share->options & HA_OPTION_PACK_RECORD) + sort_info->new_data_file_type = DYNAMIC_RECORD; + else + sort_info->new_data_file_type = STATIC_RECORD; + + /* Set delete_function for sort_delete_record() */ + memcpy((char*) &tmp, share, sizeof(*share)); + tmp.options= ~HA_OPTION_COMPRESS_RECORD; + mi_setup_functions(&tmp); + share->delete_record=tmp.delete_record; + } +} + + +int mi_make_backup_of_index(MI_INFO *info, time_t backup_time, myf flags) +{ + char backup_name[FN_REFLEN + MY_BACKUP_NAME_EXTRA_LENGTH]; + my_create_backup_name(backup_name, info->s->index_file_name, backup_time); + return my_copy(info->s->index_file_name, backup_name, flags); +} + + +static int replace_data_file(HA_CHECK *param, MI_INFO *info, File new_file) +{ + MYISAM_SHARE *share=info->s; + + mysql_file_close(new_file,MYF(0)); + info->dfile= -1; + if (param->testflag & T_BACKUP_DATA) + { + char buff[MY_BACKUP_NAME_EXTRA_LENGTH+1]; + my_create_backup_name(buff, "", param->backup_time); + my_printf_error(ER_GET_ERRMSG, + "Making backup of data file %s with extension '%s'", + MYF(ME_NOTE | ME_ERROR_LOG), share->data_file_name, + buff); + } + + /* + On Windows, the old data file cannot be deleted if it is either + open, or memory mapped. Closing the file won't remove the memory + map implicilty on Windows. We closed the data file, but we keep + the MyISAM table open. A memory map will be closed on the final + mi_close() only. So we need to unmap explicitly here. After + renaming the new file under the hook, we couldn't use the map of + the old file any more anyway. + */ + if (info->s->file_map) + { + (void) my_munmap((char*) info->s->file_map, info->s->mmaped_length); + info->s->file_map= NULL; + } + + if (change_to_newfile(share->data_file_name,MI_NAME_DEXT, + DATA_TMP_EXT, param->backup_time, + (param->testflag & T_BACKUP_DATA ? + MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) || + mi_open_datafile(info, share)) + return 1; + return 0; +} diff --git a/storage/myisam/mi_checksum.c b/storage/myisam/mi_checksum.c new file mode 100644 index 00000000..ff84fa67 --- /dev/null +++ b/storage/myisam/mi_checksum.c @@ -0,0 +1,71 @@ +/* Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Calculate a checksum for a row */ + +#include "myisamdef.h" + +ha_checksum mi_checksum(MI_INFO *info, const uchar *buf) +{ + ha_checksum crc=0; + const uchar *record= buf; + MI_COLUMNDEF *column= info->s->rec; + MI_COLUMNDEF *column_end= column+ info->s->base.fields; + my_bool skip_null_bits= MY_TEST(info->s->options & HA_OPTION_NULL_FIELDS); + + for ( ; column != column_end ; buf+= column++->length) + { + const uchar *pos; + ulong length; + + if ((record[column->null_pos] & column->null_bit) && + skip_null_bits) + continue; /* Null field */ + + switch (column->type) { + case FIELD_BLOB: + { + length=_mi_calc_blob_length(column->length- + portable_sizeof_char_ptr, + buf); + memcpy((void*) &pos, buf+column->length - portable_sizeof_char_ptr, + sizeof(char*)); + break; + } + case FIELD_VARCHAR: + { + uint pack_length= HA_VARCHAR_PACKLENGTH(column->length-1); + if (pack_length == 1) + length= (ulong) *(uchar*) buf; + else + length= uint2korr(buf); + pos= buf+pack_length; + break; + } + default: + length=column->length; + pos=buf; + break; + } + crc=my_checksum(crc, pos ? pos : (uchar*) "", length); + } + return crc; +} + + +ha_checksum mi_static_checksum(MI_INFO *info, const uchar *pos) +{ + return my_checksum(0, pos, info->s->base.reclength); +} diff --git a/storage/myisam/mi_close.c b/storage/myisam/mi_close.c new file mode 100644 index 00000000..56197729 --- /dev/null +++ b/storage/myisam/mi_close.c @@ -0,0 +1,137 @@ +/* Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* close a isam-database */ +/* + TODO: + We need to have a separate mutex on the closed file to allow other threads + to open other files during the time we flush the cache and close this file +*/ + +#include "ftdefs.h" + +int mi_close(register MI_INFO *info) +{ + int error=0,flag; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("mi_close"); + DBUG_PRINT("enter",("base: %p reopen: %u locks: %u", + info, (uint) share->reopen, + (uint) share->tot_locks)); + + if (info->open_list.data) + mysql_mutex_lock(&THR_LOCK_myisam); + if (info->lock_type == F_EXTRA_LCK) + info->lock_type=F_UNLCK; /* HA_EXTRA_NO_USER_CHANGE */ + + if (info->lock_type != F_UNLCK) + { + if (mi_lock_database(info,F_UNLCK)) + error=my_errno; + } + mysql_mutex_lock(&share->intern_lock); + + if (share->options & HA_OPTION_READ_ONLY_DATA) + { + share->r_locks--; + share->tot_locks--; + } + if (info->opt_flag & (READ_CACHE_USED | WRITE_CACHE_USED)) + { + if (end_io_cache(&info->rec_cache)) + error=my_errno; + info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED); + } + flag= !--share->reopen; + if (info->open_list.data) + myisam_open_list= list_delete(myisam_open_list, &info->open_list); + mysql_mutex_unlock(&share->intern_lock); + + my_free(mi_get_rec_buff_ptr(info, info->rec_buff)); + ftparser_call_deinitializer(info); + + if (flag) + { + DBUG_EXECUTE_IF("crash_before_flush_keys", + if (share->kfile >= 0) DBUG_SUICIDE();); + if (share->kfile >= 0 && + flush_key_blocks(share->key_cache, share->kfile, &share->dirty_part_map, + share->deleting ? FLUSH_IGNORE_CHANGED : FLUSH_RELEASE)) + error=my_errno; + if (share->kfile >= 0) + { + /* + If we are crashed, we can safely flush the current state as it will + not change the crashed state. + We can NOT write the state in other cases as other threads + may be using the file at this point IF using --external-locking. + + Also, write the state if a temporary table is not being dropped + (the server might want to reopen it, and mi_lock_database() only + writes the state for non-temp ones) + */ + if (share->mode != O_RDONLY && + (mi_is_crashed(info) || (share->temporary && !share->deleting))) + mi_state_info_write(share->kfile, &share->state, 1); + /* Decrement open count must be last I/O on this file. */ + _mi_decrement_open_count(info); + if (mysql_file_close(share->kfile, MYF(0))) + error = my_errno; + } +#ifdef HAVE_MMAP + if (share->file_map) + { + if (share->options & HA_OPTION_COMPRESS_RECORD) + _mi_unmap_file(info); + else + mi_munmap_file(info); + } +#endif + if (share->decode_trees) + { + my_free(share->decode_trees); + my_free(share->decode_tables); + } + thr_lock_delete(&share->lock); + mysql_mutex_destroy(&share->intern_lock); + { + int i,keys; + keys = share->state.header.keys; + mysql_rwlock_destroy(&share->mmap_lock); + for(i=0; i<keys; i++) { + mysql_rwlock_destroy(&share->key_root_lock[i]); + } + } + my_free(info->s); + } + if (info->open_list.data) + mysql_mutex_unlock(&THR_LOCK_myisam); + if (info->ftparser_param) + { + my_free(info->ftparser_param); + info->ftparser_param= 0; + } + if (info->dfile >= 0 && mysql_file_close(info->dfile, MYF(0))) + error = my_errno; + + myisam_log_command(MI_LOG_CLOSE,info,NULL,0,error); + my_free(info); + + if (error) + { + DBUG_RETURN(my_errno=error); + } + DBUG_RETURN(0); +} /* mi_close */ diff --git a/storage/myisam/mi_create.c b/storage/myisam/mi_create.c new file mode 100644 index 00000000..8ad6446f --- /dev/null +++ b/storage/myisam/mi_create.c @@ -0,0 +1,896 @@ +/* + Copyright (c) 2000, 2011, Oracle and/or its affiliates + Copyright (c) 2009, 2013, Monty Program Ab. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Create a MyISAM table */ + +#include "ftdefs.h" +#include "sp_defs.h" +#include <my_bit.h> + +#ifdef _WIN32 +#include <fcntl.h> +#endif +#include <m_ctype.h> + +/* + Old options is used when recreating database, from myisamchk +*/ + +int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, + uint columns, MI_COLUMNDEF *recinfo, + uint uniques, MI_UNIQUEDEF *uniquedefs, + MI_CREATE_INFO *ci,uint flags) +{ + register uint i,j; + File UNINIT_VAR(dfile),UNINIT_VAR(file); + int errpos,save_errno, create_mode= O_RDWR | O_TRUNC; + myf create_flag; + uint fields,length,max_key_length,packed,pack_bytes,pointer,real_length_diff, + key_length,info_length,key_segs,options,min_key_length_skip, + base_pos,long_varchar_count,varchar_length, + max_key_block_length,unique_key_parts,fulltext_keys,offset; + uint aligned_key_start, block_length, res; + uint internal_table= flags & HA_CREATE_INTERNAL_TABLE; + ulong reclength, real_reclength,min_pack_length; + char kfilename[FN_REFLEN],klinkname[FN_REFLEN], *klinkname_ptr= 0; + char dfilename[FN_REFLEN],dlinkname[FN_REFLEN], *dlinkname_ptr= 0; + ulong pack_reclength; + ulonglong tot_length,max_rows, tmp; + enum en_fieldtype type; + MYISAM_SHARE share; + MI_KEYDEF *keydef,tmp_keydef; + MI_UNIQUEDEF *uniquedef; + HA_KEYSEG *keyseg,tmp_keyseg; + MI_COLUMNDEF *rec; + ulong *rec_per_key_part; + my_off_t key_root[HA_MAX_POSSIBLE_KEY],key_del[MI_MAX_KEY_BLOCK_SIZE]; + MI_CREATE_INFO tmp_create_info; + DBUG_ENTER("mi_create"); + DBUG_PRINT("enter", ("keys: %u columns: %u uniques: %u flags: %u", + keys, columns, uniques, flags)); + + if (!ci) + { + bzero((char*) &tmp_create_info,sizeof(tmp_create_info)); + ci=&tmp_create_info; + } + + if (keys + uniques > MI_MAX_KEY || columns == 0) + { + DBUG_RETURN(my_errno=HA_WRONG_CREATE_OPTION); + } + + errpos=0; + options=0; + bzero((uchar*) &share,sizeof(share)); + + if (flags & HA_DONT_TOUCH_DATA) + { + if (!(ci->old_options & HA_OPTION_TEMP_COMPRESS_RECORD)) + options=ci->old_options & + (HA_OPTION_COMPRESS_RECORD | HA_OPTION_PACK_RECORD | + HA_OPTION_READ_ONLY_DATA | HA_OPTION_CHECKSUM | + HA_OPTION_TMP_TABLE | HA_OPTION_DELAY_KEY_WRITE); + else + options=ci->old_options & + (HA_OPTION_CHECKSUM | HA_OPTION_TMP_TABLE | HA_OPTION_DELAY_KEY_WRITE); + } + + if (ci->reloc_rows > ci->max_rows) + ci->reloc_rows=ci->max_rows; /* Check if wrong parameter */ + + if (!(rec_per_key_part= + (ulong*) my_malloc(mi_key_memory_MYISAM_SHARE, + (keys + uniques) * HA_MAX_KEY_SEG * sizeof(long), + MYF(MY_WME | MY_ZEROFILL)))) + DBUG_RETURN(my_errno); + + /* Start by checking fields and field-types used */ + + reclength=varchar_length=long_varchar_count=packed= + min_pack_length=pack_reclength=0; + for (rec=recinfo, fields=0 ; + fields != columns ; + rec++,fields++) + { + reclength+=rec->length; + if (rec->null_bit) + options|= HA_OPTION_NULL_FIELDS; + + if ((type=(enum en_fieldtype) rec->type) != FIELD_NORMAL && + type != FIELD_CHECK) + { + packed++; + if (type == FIELD_BLOB) + { + share.base.blobs++; + if (pack_reclength != INT_MAX32) + { + if (rec->length == 4+portable_sizeof_char_ptr) + pack_reclength= INT_MAX32; + else + pack_reclength+=(1 << ((rec->length-portable_sizeof_char_ptr)*8)); /* Max blob length */ + } + } + else if (type == FIELD_SKIP_PRESPACE || + type == FIELD_SKIP_ENDSPACE) + { + if (pack_reclength != INT_MAX32) + pack_reclength+= rec->length > 255 ? 2 : 1; + min_pack_length++; + } + else if (type == FIELD_VARCHAR) + { + varchar_length+= rec->length-1; /* Used for min_pack_length */ + packed--; + pack_reclength++; + min_pack_length++; + /* We must test for 257 as length includes pack-length */ + if (MY_TEST(rec->length >= 257)) + { + long_varchar_count++; + pack_reclength+= 2; /* May be packed on 3 bytes */ + } + options|= HA_OPTION_NULL_FIELDS; /* Use of mi_checksum() */ + } + else if (type != FIELD_SKIP_ZERO) + { + min_pack_length+=rec->length; + packed--; /* Not a pack record type */ + } + } + else /* FIELD_NORMAL */ + min_pack_length+=rec->length; + } + if ((packed & 7) == 1) + { /* Bad packing, try to remove a zero-field */ + while (rec != recinfo) + { + rec--; + if (rec->type == (int) FIELD_SKIP_ZERO && rec->length == 1) + { + /* + NOTE1: here we change a field type FIELD_SKIP_ZERO -> + FIELD_NORMAL + */ + rec->type=(int) FIELD_NORMAL; + packed--; + min_pack_length++; + break; + } + } + } + + if (packed || (flags & HA_PACK_RECORD)) + options|=HA_OPTION_PACK_RECORD; /* Must use packed records */ + /* We can't use checksum with static length rows */ + if (!(options & HA_OPTION_PACK_RECORD)) + options&= ~HA_OPTION_CHECKSUM; + if (!(options & (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD))) + min_pack_length+= varchar_length; + if (flags & HA_CREATE_TMP_TABLE) + { + options|= HA_OPTION_TMP_TABLE; + create_mode|= O_NOFOLLOW | (internal_table ? 0 : O_EXCL); + } + if (flags & HA_CREATE_CHECKSUM || (options & HA_OPTION_CHECKSUM)) + { + options|= HA_OPTION_CHECKSUM; + min_pack_length++; + } + /* + Don't set HA_OPTION_NULL_FIELDS if no checksums, as this flag makes + that file incompatible with MySQL. This is ok, as this flag is only + used if one specifics table level checksums. + */ + if (!(options & HA_OPTION_CHECKSUM)) + options&= ~HA_OPTION_NULL_FIELDS; + if (flags & HA_CREATE_DELAY_KEY_WRITE) + options|= HA_OPTION_DELAY_KEY_WRITE; + if (flags & HA_CREATE_RELIES_ON_SQL_LAYER) + options|= HA_OPTION_RELIES_ON_SQL_LAYER; + + pack_bytes= (packed+7)/8; + if (pack_reclength != INT_MAX32) + pack_reclength+= reclength+packed + + MY_TEST(test_all_bits(options, HA_OPTION_CHECKSUM | + HA_OPTION_PACK_RECORD)); + min_pack_length+= pack_bytes; + + if (!ci->data_file_length && ci->max_rows) + { + if (pack_reclength == INT_MAX32 || + (~(ulonglong) 0)/ci->max_rows < (ulonglong) pack_reclength) + ci->data_file_length= ~(ulonglong) 0; + else + ci->data_file_length=(ulonglong) ci->max_rows*pack_reclength; + } + else if (!ci->max_rows) + ci->max_rows=(ha_rows) (ci->data_file_length/(min_pack_length + + ((options & HA_OPTION_PACK_RECORD) ? + 3 : 0))); + + if (options & (HA_OPTION_COMPRESS_RECORD | HA_OPTION_PACK_RECORD)) + pointer=mi_get_pointer_length(ci->data_file_length,myisam_data_pointer_size); + else + pointer=mi_get_pointer_length(ci->max_rows,myisam_data_pointer_size); + if (!(max_rows=(ulonglong) ci->max_rows)) + max_rows= ((((ulonglong) 1 << (pointer*8)) -1) / min_pack_length); + + + real_reclength=reclength; + if (!(options & (HA_OPTION_COMPRESS_RECORD | HA_OPTION_PACK_RECORD))) + { + if (reclength <= pointer) + reclength=pointer+1; /* reserve place for delete link */ + } + else + reclength+= long_varchar_count; /* We need space for varchar! */ + + max_key_length=0; tot_length=0 ; key_segs=0; + fulltext_keys=0; + max_key_block_length=0; + share.state.rec_per_key_part=rec_per_key_part; + share.state.key_root=key_root; + share.state.key_del=key_del; + if (uniques) + { + max_key_block_length= myisam_block_size; + max_key_length= MI_UNIQUE_HASH_LENGTH + pointer; + } + + for (i=0, keydef=keydefs ; i < keys ; i++ , keydef++) + { + + share.state.key_root[i]= HA_OFFSET_ERROR; + min_key_length_skip=length=real_length_diff=0; + key_length=pointer; + if (keydef->flag & HA_SPATIAL) + { +#ifdef HAVE_SPATIAL + /* BAR TODO to support 3D and more dimensions in the future */ + uint sp_segs=SPDIMS*2; + keydef->flag=HA_SPATIAL; + + if (flags & HA_DONT_TOUCH_DATA) + { + /* + called by myisamchk - i.e. table structure was taken from + MYI file and SPATIAL key *does have* additional sp_segs keysegs. + keydef->seg here points right at the GEOMETRY segment, + so we only need to decrease keydef->keysegs. + (see recreate_table() in mi_check.c) + */ + keydef->keysegs= 1; + } + + for (j=0, keyseg=keydef->seg ; (int) j < keydef->keysegs ; + j++, keyseg++) + { + if (keyseg->type != HA_KEYTYPE_BINARY && + keyseg->type != HA_KEYTYPE_VARBINARY1 && + keyseg->type != HA_KEYTYPE_VARBINARY2) + { + my_errno=HA_WRONG_CREATE_OPTION; + goto err_no_lock; + } + } + DBUG_ASSERT(keydef->keysegs == 1); + keydef->keysegs= sp_segs + 1; + key_length+=SPLEN*sp_segs; + length++; /* At least one length byte */ + min_key_length_skip+=SPLEN*2*SPDIMS; +#else + my_errno= HA_ERR_UNSUPPORTED; + goto err_no_lock; +#endif /*HAVE_SPATIAL*/ + } + else if (keydef->flag & HA_FULLTEXT) + { + keydef->flag=HA_FULLTEXT | HA_PACK_KEY | HA_VAR_LENGTH_KEY; + options|=HA_OPTION_PACK_KEYS; /* Using packed keys */ + + for (j=0, keyseg=keydef->seg ; (int) j < keydef->keysegs ; + j++, keyseg++) + { + if (keyseg->type != HA_KEYTYPE_TEXT && + keyseg->type != HA_KEYTYPE_VARTEXT1 && + keyseg->type != HA_KEYTYPE_VARTEXT2) + { + my_errno=HA_WRONG_CREATE_OPTION; + goto err_no_lock; + } + if (!(keyseg->flag & HA_BLOB_PART) && + (keyseg->type == HA_KEYTYPE_VARTEXT1 || + keyseg->type == HA_KEYTYPE_VARTEXT2)) + { + /* Make a flag that this is a VARCHAR */ + keyseg->flag|= HA_VAR_LENGTH_PART; + /* Store in bit_start number of bytes used to pack the length */ + keyseg->bit_start= ((keyseg->type == HA_KEYTYPE_VARTEXT1)? + 1 : 2); + } + } + + fulltext_keys++; + key_length+= HA_FT_MAXBYTELEN+HA_FT_WLEN; + length++; /* At least one length byte */ + min_key_length_skip+=HA_FT_MAXBYTELEN; + real_length_diff=HA_FT_MAXBYTELEN-FT_MAX_WORD_LEN_FOR_SORT; + } + else + { + /* Test if prefix compression */ + if (keydef->flag & HA_PACK_KEY) + { + /* Can't use space_compression on number keys */ + if ((keydef->seg[0].flag & HA_SPACE_PACK) && + keydef->seg[0].type == (int) HA_KEYTYPE_NUM) + keydef->seg[0].flag&= ~HA_SPACE_PACK; + + /* Only use HA_PACK_KEY when first segment is a variable length key */ + if (!(keydef->seg[0].flag & (HA_SPACE_PACK | HA_BLOB_PART | + HA_VAR_LENGTH_PART))) + { + /* pack relative to previous key */ + keydef->flag&= ~HA_PACK_KEY; + keydef->flag|= HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY; + } + else + { + keydef->seg[0].flag|=HA_PACK_KEY; /* for easyer intern test */ + keydef->flag|=HA_VAR_LENGTH_KEY; + options|=HA_OPTION_PACK_KEYS; /* Using packed keys */ + } + } + if (keydef->flag & HA_BINARY_PACK_KEY) + options|=HA_OPTION_PACK_KEYS; /* Using packed keys */ + + if (keydef->flag & HA_AUTO_KEY && ci->with_auto_increment) + share.base.auto_key=i+1; + for (j=0, keyseg=keydef->seg ; j < keydef->keysegs ; j++, keyseg++) + { + /* numbers are stored with high by first to make compression easier */ + switch (keyseg->type) { + case HA_KEYTYPE_SHORT_INT: + case HA_KEYTYPE_LONG_INT: + case HA_KEYTYPE_FLOAT: + case HA_KEYTYPE_DOUBLE: + case HA_KEYTYPE_USHORT_INT: + case HA_KEYTYPE_ULONG_INT: + case HA_KEYTYPE_LONGLONG: + case HA_KEYTYPE_ULONGLONG: + case HA_KEYTYPE_INT24: + case HA_KEYTYPE_UINT24: + case HA_KEYTYPE_INT8: + keyseg->flag|= HA_SWAP_KEY; + break; + case HA_KEYTYPE_VARTEXT1: + case HA_KEYTYPE_VARTEXT2: + case HA_KEYTYPE_VARBINARY1: + case HA_KEYTYPE_VARBINARY2: + if (!(keyseg->flag & HA_BLOB_PART)) + { + /* Make a flag that this is a VARCHAR */ + keyseg->flag|= HA_VAR_LENGTH_PART; + /* Store in bit_start number of bytes used to pack the length */ + keyseg->bit_start= ((keyseg->type == HA_KEYTYPE_VARTEXT1 || + keyseg->type == HA_KEYTYPE_VARBINARY1) ? + 1 : 2); + } + break; + default: + break; + } + if (keyseg->flag & HA_SPACE_PACK) + { + DBUG_ASSERT(!(keyseg->flag & HA_VAR_LENGTH_PART)); + keydef->flag |= HA_SPACE_PACK_USED | HA_VAR_LENGTH_KEY; + options|=HA_OPTION_PACK_KEYS; /* Using packed keys */ + length++; /* At least one length byte */ + min_key_length_skip+=keyseg->length; + if (keyseg->length >= 255) + { /* prefix may be 3 bytes */ + min_key_length_skip+=2; + length+=2; + } + } + if (keyseg->flag & (HA_VAR_LENGTH_PART | HA_BLOB_PART)) + { + DBUG_ASSERT(!test_all_bits(keyseg->flag, + (HA_VAR_LENGTH_PART | HA_BLOB_PART))); + keydef->flag|=HA_VAR_LENGTH_KEY; + length++; /* At least one length byte */ + options|=HA_OPTION_PACK_KEYS; /* Using packed keys */ + min_key_length_skip+=keyseg->length; + if (keyseg->length >= 255) + { /* prefix may be 3 bytes */ + min_key_length_skip+=2; + length+=2; + } + } + key_length+= keyseg->length; + if (keyseg->null_bit) + { + key_length++; + options|=HA_OPTION_PACK_KEYS; + keyseg->flag|=HA_NULL_PART; + keydef->flag|=HA_VAR_LENGTH_KEY | HA_NULL_PART_KEY; + } + } + } /* if HA_FULLTEXT */ + key_segs+=keydef->keysegs; + if (keydef->keysegs > HA_MAX_KEY_SEG) + { + my_errno=HA_WRONG_CREATE_OPTION; + goto err_no_lock; + } + /* + key_segs may be 0 in the case when we only want to be able to + add on row into the table. This can happen with some DISTINCT queries + in MySQL + */ + if ((keydef->flag & (HA_NOSAME | HA_NULL_PART_KEY)) == HA_NOSAME && + key_segs) + share.state.rec_per_key_part[key_segs-1]=1L; + length+=key_length; + /* Get block length for key, if defined by user */ + block_length= (keydef->block_length ? + my_round_up_to_next_power(keydef->block_length) : + myisam_block_size); + block_length= MY_MAX(block_length, MI_MIN_KEY_BLOCK_LENGTH); + block_length= MY_MIN(block_length, MI_MAX_KEY_BLOCK_LENGTH); + + keydef->block_length= (uint16) MI_BLOCK_SIZE(length-real_length_diff, + pointer,MI_MAX_KEYPTR_SIZE, + block_length); + if (keydef->block_length > MI_MAX_KEY_BLOCK_LENGTH || + length >= HA_MAX_KEY_BUFF) + { + my_errno=HA_WRONG_CREATE_OPTION; + goto err_no_lock; + } + set_if_bigger(max_key_block_length,keydef->block_length); + keydef->keylength= (uint16) key_length; + keydef->minlength= (uint16) (length-min_key_length_skip); + keydef->maxlength= (uint16) length; + + if (length > max_key_length) + max_key_length= length; + tot_length+= (max_rows/(ulong) (((uint) keydef->block_length-5)/ + (length*2)))* + (ulong) keydef->block_length; + } + for (i=max_key_block_length/MI_MIN_KEY_BLOCK_LENGTH ; i-- ; ) + key_del[i]=HA_OFFSET_ERROR; + + unique_key_parts=0; + for (i=0, uniquedef=uniquedefs ; i < uniques ; i++ , uniquedef++) + { + uniquedef->key=keys+i; + unique_key_parts+=uniquedef->keysegs; + share.state.key_root[keys+i]= HA_OFFSET_ERROR; + tot_length+= (max_rows/(ulong) (((uint) myisam_block_size-5)/ + ((MI_UNIQUE_HASH_LENGTH + pointer)*2)))* + (ulong) myisam_block_size; + } + keys+=uniques; /* Each unique has 1 key */ + key_segs+=uniques; /* Each unique has 1 key seg */ + + base_pos=(MI_STATE_INFO_SIZE + keys * MI_STATE_KEY_SIZE + + max_key_block_length/MI_MIN_KEY_BLOCK_LENGTH* + MI_STATE_KEYBLOCK_SIZE+ + key_segs*MI_STATE_KEYSEG_SIZE); + info_length=base_pos+(uint) (MI_BASE_INFO_SIZE+ + keys * MI_KEYDEF_SIZE+ + uniques * MI_UNIQUEDEF_SIZE + + (key_segs + unique_key_parts)*HA_KEYSEG_SIZE+ + columns*MI_COLUMNDEF_SIZE); + DBUG_PRINT("info", ("info_length: %u", info_length)); + /* There are only 16 bits for the total header length. */ + if (info_length > 65535) + { + my_printf_error(HA_WRONG_CREATE_OPTION, + "MyISAM table '%s' has too many columns and/or " + "indexes and/or unique constraints.", + MYF(0), name + dirname_length(name)); + my_errno= HA_WRONG_CREATE_OPTION; + goto err_no_lock; + } + + bmove(share.state.header.file_version,(uchar*) myisam_file_magic,4); + ci->old_options=options| (ci->old_options & HA_OPTION_TEMP_COMPRESS_RECORD ? + HA_OPTION_COMPRESS_RECORD | + HA_OPTION_TEMP_COMPRESS_RECORD: 0); + mi_int2store(share.state.header.options,ci->old_options); + mi_int2store(share.state.header.header_length,info_length); + mi_int2store(share.state.header.state_info_length,MI_STATE_INFO_SIZE); + mi_int2store(share.state.header.base_info_length,MI_BASE_INFO_SIZE); + mi_int2store(share.state.header.base_pos,base_pos); + share.state.header.language= (ci->language ? + ci->language : default_charset_info->number); + share.state.header.max_block_size_index= max_key_block_length/MI_MIN_KEY_BLOCK_LENGTH; + + share.state.dellink = HA_OFFSET_ERROR; + share.state.process= (ulong) getpid(); + share.state.unique= (ulong) 0; + share.state.update_count=(ulong) 0; + share.state.version= (ulong) time((time_t*) 0); + share.state.sortkey= (ushort) ~0; + share.state.auto_increment=ci->auto_increment; + share.options=options; + share.base.rec_reflength=pointer; + /* Get estimate for index file length (this may be wrong for FT keys) */ + tmp= (tot_length + max_key_block_length * keys * + MI_INDEX_BLOCK_MARGIN) / MI_MIN_KEY_BLOCK_LENGTH; + /* + use maximum of key_file_length we calculated and key_file_length value we + got from MYI file header (see also myisampack.c:save_state) + */ + share.base.key_reflength= + mi_get_pointer_length(MY_MAX(ci->key_file_length,tmp),3); + share.base.keys= share.state.header.keys= keys; + share.state.header.uniques= uniques; + share.state.header.fulltext_keys= fulltext_keys; + mi_int2store(share.state.header.key_parts,key_segs); + mi_int2store(share.state.header.unique_key_parts,unique_key_parts); + + mi_set_all_keys_active(share.state.key_map, keys); + aligned_key_start= my_round_up_to_next_power(max_key_block_length ? + max_key_block_length : + myisam_block_size); + + share.base.keystart= share.state.state.key_file_length= + MY_ALIGN(info_length, aligned_key_start); + share.base.max_key_block_length=max_key_block_length; + share.base.max_key_length=ALIGN_SIZE(max_key_length+4); + share.base.records=ci->max_rows; + share.base.reloc= ci->reloc_rows; + share.base.reclength=real_reclength; + share.base.pack_reclength= reclength + MY_TEST(options & HA_OPTION_CHECKSUM); + share.base.max_pack_length=pack_reclength; + share.base.min_pack_length=min_pack_length; + share.base.pack_bits= pack_bytes; + share.base.fields=fields; + share.base.pack_fields=packed; + + /* max_data_file_length and max_key_file_length are recalculated on open */ + if (options & HA_OPTION_TMP_TABLE) + share.base.max_data_file_length=(my_off_t) ci->data_file_length; + + share.base.min_block_length= + (share.base.pack_reclength+3 < MI_EXTEND_BLOCK_LENGTH && + ! share.base.blobs) ? + MY_MAX(share.base.pack_reclength,MI_MIN_BLOCK_LENGTH) : + MI_EXTEND_BLOCK_LENGTH; + if (! (flags & HA_DONT_TOUCH_DATA)) + share.state.create_time= time((time_t*) 0); + + if (!internal_table) + mysql_mutex_lock(&THR_LOCK_myisam); + + /* + NOTE: For test_if_reopen() we need a real path name. Hence we need + MY_RETURN_REAL_PATH for every fn_format(filename, ...). + */ + if (ci->index_file_name) + { + char *iext= strrchr(ci->index_file_name, '.'); + int have_iext= iext && !strcmp(iext, MI_NAME_IEXT); + if (options & HA_OPTION_TMP_TABLE) + { + char *path; + /* chop off the table name, tempory tables use generated name */ + if ((path= strrchr(ci->index_file_name, FN_LIBCHAR))) + *path= '\0'; + fn_format(kfilename, name, ci->index_file_name, MI_NAME_IEXT, + MY_REPLACE_DIR | MY_UNPACK_FILENAME | + MY_RETURN_REAL_PATH | MY_APPEND_EXT); + } + else + { + fn_format(kfilename, ci->index_file_name, "", MI_NAME_IEXT, + MY_UNPACK_FILENAME | MY_RETURN_REAL_PATH | + (have_iext ? MY_REPLACE_EXT : MY_APPEND_EXT)); + } + fn_format(klinkname, name, "", MI_NAME_IEXT, + MY_UNPACK_FILENAME|MY_APPEND_EXT); + klinkname_ptr= klinkname; + /* + Don't create the table if the link or file exists to ensure that one + doesn't accidentally destroy another table. + */ + create_flag=0; + } + else + { + char *iext= strrchr(name, '.'); + int have_iext= iext && !strcmp(iext, MI_NAME_IEXT); + fn_format(kfilename, name, "", MI_NAME_IEXT, MY_UNPACK_FILENAME | + (internal_table ? 0 : MY_RETURN_REAL_PATH) | + (have_iext ? MY_REPLACE_EXT : MY_APPEND_EXT)); + klinkname_ptr= 0; + /* Replace the current file */ + create_flag=(flags & HA_CREATE_KEEP_FILES) ? 0 : MY_DELETE_OLD; + } + + /* + If a MRG_MyISAM table is in use, the mapped MyISAM tables are open, + but no entry is made in the table cache for them. + A TRUNCATE command checks for the table in the cache only and could + be fooled to believe, the table is not open. + Pull the emergency brake in this situation. (Bug #8306) + + NOTE: The filename is compared against unique_file_name of every + open table. Hence we need a real path here. + */ + if (!internal_table && test_if_reopen(kfilename)) + { + my_printf_error(HA_ERR_TABLE_EXIST, "MyISAM table '%s' is in use " + "(most likely by a MERGE table). Try FLUSH TABLES.", + MYF(0), name + dirname_length(name)); + my_errno= HA_ERR_TABLE_EXIST; + goto err; + } + + if ((file= mysql_file_create_with_symlink(mi_key_file_kfile, + klinkname_ptr, kfilename, 0, + create_mode, + MYF(MY_WME | create_flag))) < 0) + goto err; + errpos=1; + + if (!(flags & HA_DONT_TOUCH_DATA)) + { + { + if (ci->data_file_name) + { + char *dext= strrchr(ci->data_file_name, '.'); + int have_dext= dext && !strcmp(dext, MI_NAME_DEXT); + + if (options & HA_OPTION_TMP_TABLE) + { + char *path; + /* chop off the table name, tempory tables use generated name */ + if ((path= strrchr(ci->data_file_name, FN_LIBCHAR))) + *path= '\0'; + fn_format(dfilename, name, ci->data_file_name, MI_NAME_DEXT, + MY_REPLACE_DIR | MY_UNPACK_FILENAME | MY_APPEND_EXT); + } + else + { + fn_format(dfilename, ci->data_file_name, "", MI_NAME_DEXT, + MY_UNPACK_FILENAME | + (have_dext ? MY_REPLACE_EXT : MY_APPEND_EXT)); + } + + fn_format(dlinkname, name, "",MI_NAME_DEXT, + MY_UNPACK_FILENAME | MY_APPEND_EXT); + dlinkname_ptr= dlinkname; + create_flag=0; + } + else + { + fn_format(dfilename,name,"", MI_NAME_DEXT, + MY_UNPACK_FILENAME | MY_APPEND_EXT); + dlinkname_ptr= 0; + create_flag=(flags & HA_CREATE_KEEP_FILES) ? 0 : MY_DELETE_OLD; + } + if ((dfile= + mysql_file_create_with_symlink(mi_key_file_dfile, + dlinkname_ptr, dfilename, 0, + create_mode, + MYF(MY_WME | create_flag))) < 0) + goto err; + } + errpos=3; + } + + DBUG_PRINT("info", ("write state info and base info")); + if (mi_state_info_write(file, &share.state, 2) || + mi_base_info_write(file, &share.base)) + goto err; +#ifdef DBUG_TRACE + if ((uint) mysql_file_tell(file, MYF(0)) != base_pos + MI_BASE_INFO_SIZE) + { + uint pos=(uint) mysql_file_tell(file, MYF(0)); + DBUG_PRINT("warning",("base_length: %d != used_length: %d", + base_pos+ MI_BASE_INFO_SIZE, pos)); + } +#endif + + /* Write key and keyseg definitions + + TODO: update key and keyseg definitions for inplace alter (grep sql layer by + MDEV-25803). Do the same for Aria. + */ + DBUG_PRINT("info", ("write key and keyseg definitions")); + for (i=0 ; i < share.base.keys - uniques; i++) + { + uint sp_segs=(keydefs[i].flag & HA_SPATIAL) ? 2*SPDIMS : 0; + + if (mi_keydef_write(file, &keydefs[i])) + goto err; + for (j=0 ; j < keydefs[i].keysegs-sp_segs ; j++) + if (mi_keyseg_write(file, &keydefs[i].seg[j])) + goto err; +#ifdef HAVE_SPATIAL + for (j=0 ; j < sp_segs ; j++) + { + HA_KEYSEG sseg; + sseg.type=SPTYPE; + sseg.language= 7; /* Binary */ + sseg.null_bit=0; + sseg.bit_start=0; + sseg.bit_length= 0; + sseg.bit_pos= 0; + sseg.length=SPLEN; + sseg.null_pos=0; + sseg.start=j*SPLEN; + sseg.flag= HA_SWAP_KEY; + if (mi_keyseg_write(file, &sseg)) + goto err; + } +#endif + } + /* Create extra keys for unique definitions */ + offset= real_reclength - uniques * MI_UNIQUE_HASH_LENGTH; + bzero((char*) &tmp_keydef,sizeof(tmp_keydef)); + bzero((char*) &tmp_keyseg,sizeof(tmp_keyseg)); + for (i=0; i < uniques ; i++) + { + tmp_keydef.keysegs=1; + tmp_keydef.block_length= (uint16)myisam_block_size; + tmp_keydef.keylength= MI_UNIQUE_HASH_LENGTH + pointer; + tmp_keydef.minlength=tmp_keydef.maxlength=tmp_keydef.keylength; + tmp_keyseg.type= MI_UNIQUE_HASH_TYPE; + tmp_keyseg.length= MI_UNIQUE_HASH_LENGTH; + tmp_keyseg.start= offset; + offset+= MI_UNIQUE_HASH_LENGTH; + if (mi_keydef_write(file,&tmp_keydef) || + mi_keyseg_write(file,(&tmp_keyseg))) + goto err; + } + + /* Save unique definition */ + DBUG_PRINT("info", ("write unique definitions")); + for (i=0 ; i < share.state.header.uniques ; i++) + { + HA_KEYSEG *keyseg_end; + keyseg= uniquedefs[i].seg; + if (mi_uniquedef_write(file, &uniquedefs[i])) + goto err; + for (keyseg= uniquedefs[i].seg, keyseg_end= keyseg+ uniquedefs[i].keysegs; + keyseg < keyseg_end; + keyseg++) + { + switch (keyseg->type) { + case HA_KEYTYPE_VARTEXT1: + case HA_KEYTYPE_VARTEXT2: + case HA_KEYTYPE_VARBINARY1: + case HA_KEYTYPE_VARBINARY2: + if (!(keyseg->flag & HA_BLOB_PART)) + { + keyseg->flag|= HA_VAR_LENGTH_PART; + keyseg->bit_start= ((keyseg->type == HA_KEYTYPE_VARTEXT1 || + keyseg->type == HA_KEYTYPE_VARBINARY1) ? + 1 : 2); + } + break; + default: + break; + } + if (mi_keyseg_write(file, keyseg)) + goto err; + } + } + DBUG_PRINT("info", ("write field definitions")); + for (i=0 ; i < share.base.fields ; i++) + if (mi_recinfo_write(file, &recinfo[i])) + goto err; + +#ifdef DBUG_TRACE + { + uint pos= (uint) mysql_file_tell(file, MYF(0)); + if (pos != info_length) + DBUG_PRINT("warning",("info_length: %d != used_length: %d", + info_length, pos)); + } +#endif + + /* Enlarge files */ + DBUG_PRINT("info", ("enlarge to keystart: %lu", (ulong) share.base.keystart)); + if (mysql_file_chsize(file, (ulong) share.base.keystart, 0, MYF(0))) + goto err; + + if (! (flags & HA_DONT_TOUCH_DATA)) + { +#ifdef USE_RELOC + if (mysql_file_chsize(dfile, share.base.min_pack_length*ci->reloc_rows, + 0, MYF(0))) + goto err; +#endif + errpos=2; + if (mysql_file_close(dfile, MYF(0))) + goto err; + } + errpos=0; + if (!internal_table) + mysql_mutex_unlock(&THR_LOCK_myisam); + res= 0; + if (mysql_file_close(file, MYF(0))) + res= my_errno; + my_free(rec_per_key_part); + DBUG_RETURN(res); + +err: + if (!internal_table) + mysql_mutex_unlock(&THR_LOCK_myisam); + +err_no_lock: + save_errno=my_errno; + switch (errpos) { + case 3: + (void) mysql_file_close(dfile, MYF(0)); + /* fall through */ + case 2: + if (! (flags & HA_DONT_TOUCH_DATA)) + { + mysql_file_delete(mi_key_file_dfile, dfilename, MYF(0)); + if (dlinkname_ptr) + mysql_file_delete(mi_key_file_dfile, dlinkname_ptr, MYF(0)); + } + /* fall through */ + case 1: + (void) mysql_file_close(file, MYF(0)); + if (! (flags & HA_DONT_TOUCH_DATA)) + { + mysql_file_delete(mi_key_file_kfile, kfilename, MYF(0)); + if (klinkname_ptr) + mysql_file_delete(mi_key_file_kfile, klinkname_ptr, MYF(0)); + } + } + my_free(rec_per_key_part); + DBUG_RETURN(my_errno=save_errno); /* return the fatal errno */ +} + + +uint mi_get_pointer_length(ulonglong file_length, uint def) +{ + DBUG_ASSERT(def >= 2 && def <= 7); + if (file_length) /* If not default */ + { +#ifdef NOT_YET_READY_FOR_8_BYTE_POINTERS + if (file_length >= 1ULL << 56) + def=8; + else +#endif + if (file_length >= 1ULL << 48) + def=7; + else if (file_length >= 1ULL << 40) + def=6; + else if (file_length >= 1ULL << 32) + def=5; + else if (file_length >= 1ULL << 24) + def=4; + else if (file_length >= 1ULL << 16) + def=3; + else + def=2; + } + return def; +} diff --git a/storage/myisam/mi_dbug.c b/storage/myisam/mi_dbug.c new file mode 100644 index 00000000..6f99d074 --- /dev/null +++ b/storage/myisam/mi_dbug.c @@ -0,0 +1,207 @@ +/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Support rutiner with are using with dbug */ + +#include "myisamdef.h" + + /* Print a key in user understandable format */ + +void _mi_print_key(FILE *stream, register HA_KEYSEG *keyseg, + const uchar *key, uint length) +{ + int flag; + short int s_1; + long int l_1; + float f_1; + double d_1; + const uchar *end; + const uchar *key_end=key+length; + + (void) fputs("Key: \"",stream); + flag=0; + for (; keyseg->type && key < key_end ;keyseg++) + { + if (flag++) + (void) putc('-',stream); + end= key+ keyseg->length; + if (keyseg->flag & HA_NULL_PART) + { + /* A NULL value is encoded by a 1-byte flag. Zero means NULL. */ + if (! *(key++)) + { + fprintf(stream,"NULL"); + continue; + } + end++; + } + + switch (keyseg->type) { + case HA_KEYTYPE_BINARY: + if (!(keyseg->flag & HA_SPACE_PACK) && keyseg->length == 1) + { /* packed binary digit */ + (void) fprintf(stream,"%d",(uint) *key++); + break; + } + /* fall through */ + case HA_KEYTYPE_TEXT: + case HA_KEYTYPE_NUM: + if (keyseg->flag & HA_SPACE_PACK) + { + (void) fprintf(stream,"%.*s",(int) *key,key+1); + key+= (int) *key+1; + } + else + { + (void) fprintf(stream,"%.*s",(int) keyseg->length,key); + key=end; + } + break; + case HA_KEYTYPE_INT8: + (void) fprintf(stream,"%d",(int) *((signed char*) key)); + key=end; + break; + case HA_KEYTYPE_SHORT_INT: + s_1= mi_sint2korr(key); + (void) fprintf(stream,"%d",(int) s_1); + key=end; + break; + case HA_KEYTYPE_USHORT_INT: + { + ushort u_1; + u_1= mi_uint2korr(key); + (void) fprintf(stream,"%u",(uint) u_1); + key=end; + break; + } + case HA_KEYTYPE_LONG_INT: + l_1=mi_sint4korr(key); + (void) fprintf(stream,"%ld",l_1); + key=end; + break; + case HA_KEYTYPE_ULONG_INT: + l_1=mi_uint4korr(key); + (void) fprintf(stream,"%lu",(ulong) l_1); + key=end; + break; + case HA_KEYTYPE_INT24: + (void) fprintf(stream,"%ld",(long) mi_sint3korr(key)); + key=end; + break; + case HA_KEYTYPE_UINT24: + (void) fprintf(stream,"%lu",(ulong) mi_uint3korr(key)); + key=end; + break; + case HA_KEYTYPE_FLOAT: + mi_float4get(f_1,key); + (void) fprintf(stream,"%g",(double) f_1); + key=end; + break; + case HA_KEYTYPE_DOUBLE: + mi_float8get(d_1,key); + (void) fprintf(stream,"%g",d_1); + key=end; + break; +#ifdef HAVE_LONG_LONG + case HA_KEYTYPE_LONGLONG: + { + char buff[21]; + longlong10_to_str(mi_sint8korr(key),buff,-10); + (void) fprintf(stream,"%s",buff); + key=end; + break; + } + case HA_KEYTYPE_ULONGLONG: + { + char buff[21]; + longlong10_to_str(mi_sint8korr(key),buff,10); + (void) fprintf(stream,"%s",buff); + key=end; + break; + } +#endif + case HA_KEYTYPE_BIT: + { + uint i; + fputs("0x",stream); + for (i=0 ; i < keyseg->length ; i++) + fprintf(stream, "%02x", (uint) *key++); + key= end; + break; + } + case HA_KEYTYPE_VARTEXT1: /* VARCHAR and TEXT */ + case HA_KEYTYPE_VARTEXT2: /* VARCHAR and TEXT */ + case HA_KEYTYPE_VARBINARY1: /* VARBINARY and BLOB */ + case HA_KEYTYPE_VARBINARY2: /* VARBINARY and BLOB */ + { + uint tmp_length; + get_key_length(tmp_length,key); + /* + The following command sometimes gives a warning from valgrind. + Not yet sure if the bug is in valgrind, glibc or mysqld + */ + (void) fprintf(stream,"%.*s",(int) tmp_length,key); + key+=tmp_length; + break; + } + default: break; /* This never happens */ + } + } + (void) fputs("\"\n",stream); + return; +} /* print_key */ + + +#ifdef EXTRA_DEBUG +/** + Check if the named table is in the open list. + + @param[in] name table path as in MYISAM_SHARE::unique_file_name + @param[in] where verbal description of caller + + @retval TRUE table is in open list + @retval FALSE table is not in open list + + @note This function takes THR_LOCK_myisam. Do not call it when + this mutex is locked by this thread already. +*/ + +my_bool check_table_is_closed(const char *name, const char *where) +{ + char filename[FN_REFLEN]; + LIST *pos; + DBUG_ENTER("check_table_is_closed"); + + (void) fn_format(filename,name,"",MI_NAME_IEXT,4+16+32); + mysql_mutex_lock(&THR_LOCK_myisam); + for (pos=myisam_open_list ; pos ; pos=pos->next) + { + MI_INFO *info=(MI_INFO*) pos->data; + MYISAM_SHARE *share=info->s; + if (!strcmp(share->unique_file_name,filename)) + { + if (share->last_version) + { + mysql_mutex_unlock(&THR_LOCK_myisam); + fprintf(stderr,"Warning: Table: %s is open on %s\n", name,where); + DBUG_PRINT("warning",("Table: %s is open on %s", name,where)); + DBUG_RETURN(1); + } + } + } + mysql_mutex_unlock(&THR_LOCK_myisam); + DBUG_RETURN(0); +} +#endif /* EXTRA_DEBUG */ diff --git a/storage/myisam/mi_delete.c b/storage/myisam/mi_delete.c new file mode 100644 index 00000000..62409a15 --- /dev/null +++ b/storage/myisam/mi_delete.c @@ -0,0 +1,901 @@ +/* Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Remove a row from a MyISAM table */ + +#include "fulltext.h" +#include "rt_index.h" + +static int d_search(MI_INFO *info,MI_KEYDEF *keyinfo,uint comp_flag, + uchar *key,uint key_length,my_off_t page,uchar *anc_buff); +static int del(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *key,uchar *anc_buff, + my_off_t leaf_page,uchar *leaf_buff,uchar *keypos, + my_off_t next_block,uchar *ret_key); +static int underflow(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *anc_buff, + my_off_t leaf_page,uchar *leaf_buff,uchar *keypos); +static uint remove_key(MI_KEYDEF *keyinfo,uint nod_flag,uchar *keypos, + uchar *lastkey,uchar *page_end, + my_off_t *next_block); +static int _mi_ck_real_delete(register MI_INFO *info,MI_KEYDEF *keyinfo, + uchar *key, uint key_length, my_off_t *root); + + +int mi_delete(MI_INFO *info,const uchar *record) +{ + uint i; + uchar *old_key; + int save_errno; + char lastpos[8]; + + MYISAM_SHARE *share=info->s; + DBUG_ENTER("mi_delete"); + + /* Test if record is in datafile */ + + DBUG_EXECUTE_IF("myisam_pretend_crashed_table_on_usage", + mi_print_error(info->s, HA_ERR_CRASHED); + DBUG_RETURN(my_errno= HA_ERR_CRASHED);); + DBUG_EXECUTE_IF("my_error_test_undefined_error", + mi_print_error(info->s, INT_MAX); + DBUG_RETURN(my_errno= INT_MAX);); + if (!(info->update & HA_STATE_AKTIV)) + { + DBUG_RETURN(my_errno=HA_ERR_KEY_NOT_FOUND); /* No database read */ + } + if (share->options & HA_OPTION_READ_ONLY_DATA) + { + DBUG_RETURN(my_errno=EACCES); + } + if (_mi_readinfo(info,F_WRLCK,1)) + DBUG_RETURN(my_errno); + if (info->s->calc_checksum) + info->checksum=(*info->s->calc_checksum)(info,record); + if ((*share->compare_record)(info,record)) + goto err; /* Error on read-check */ + + if (_mi_mark_file_changed(info)) + goto err; + + /* Remove all keys from the .ISAM file */ + + old_key=info->lastkey2; + for (i=0 ; i < share->base.keys ; i++ ) + { + if (mi_is_key_active(info->s->state.key_map, i)) + { + info->s->keyinfo[i].version++; + if (info->s->keyinfo[i].flag & HA_FULLTEXT ) + { + if (_mi_ft_del(info,i, old_key,record,info->lastpos)) + goto err; + } + else + { + if (info->s->keyinfo[i].ck_delete(info,i,old_key, + _mi_make_key(info,i,old_key,record,info->lastpos))) + goto err; + } + /* The above changed info->lastkey2. Inform mi_rnext_same(). */ + info->update&= ~HA_STATE_RNEXT_SAME; + } + } + + if ((*share->delete_record)(info)) + goto err; /* Remove record from database */ + info->state->checksum-=info->checksum; + + info->update= HA_STATE_CHANGED+HA_STATE_DELETED+HA_STATE_ROW_CHANGED; + info->state->records--; + + mi_sizestore(lastpos,info->lastpos); + myisam_log_command(MI_LOG_DELETE,info,(uchar*) lastpos,sizeof(lastpos),0); + (void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE); + + if (info->invalidator != 0) + { + DBUG_PRINT("info", ("invalidator... '%s' (delete)", info->filename)); + (*info->invalidator)(info->filename); + info->invalidator=0; + } + DBUG_RETURN(0); + +err: + save_errno=my_errno; + mi_sizestore(lastpos,info->lastpos); + myisam_log_command(MI_LOG_DELETE,info,(uchar*) lastpos, sizeof(lastpos),0); + if (save_errno != HA_ERR_RECORD_CHANGED) + { + mi_print_error(info->s, HA_ERR_CRASHED); + mi_mark_crashed(info); /* mark table crashed */ + } + (void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE); + info->update|=HA_STATE_WRITTEN; /* Buffer changed */ + my_errno=save_errno; + if (save_errno == HA_ERR_KEY_NOT_FOUND) + { + mi_print_error(info->s, HA_ERR_CRASHED); + my_errno=HA_ERR_CRASHED; + } + + DBUG_RETURN(my_errno); +} /* mi_delete */ + + + /* Remove a key from the btree index */ + +int _mi_ck_delete(register MI_INFO *info, uint keynr, uchar *key, + uint key_length) +{ + return _mi_ck_real_delete(info, info->s->keyinfo+keynr, key, key_length, + &info->s->state.key_root[keynr]); +} /* _mi_ck_delete */ + + +static int _mi_ck_real_delete(register MI_INFO *info, MI_KEYDEF *keyinfo, + uchar *key, uint key_length, my_off_t *root) +{ + int error; + uint nod_flag; + my_off_t old_root; + uchar *root_buff; + DBUG_ENTER("_mi_ck_real_delete"); + + if ((old_root=*root) == HA_OFFSET_ERROR) + { + mi_print_error(info->s, HA_ERR_CRASHED); + DBUG_RETURN(my_errno=HA_ERR_CRASHED); + } + if (!(root_buff= (uchar*) my_alloca((uint) keyinfo->block_length+ + HA_MAX_KEY_BUFF*2))) + { + DBUG_PRINT("error",("Couldn't allocate memory")); + DBUG_RETURN(my_errno=ENOMEM); + } + DBUG_PRINT("info",("root_page: %ld", (long) old_root)); + if (!_mi_fetch_keypage(info,keyinfo,old_root,DFLT_INIT_HITS,root_buff,0)) + { + error= -1; + goto err; + } + if ((error=d_search(info,keyinfo, + (keyinfo->flag & HA_FULLTEXT ? + SEARCH_FIND | SEARCH_UPDATE | SEARCH_INSERT : + SEARCH_SAME), + key,key_length,old_root,root_buff)) >0) + { + if (error == 2) + { + DBUG_PRINT("test",("Enlarging of root when deleting")); + error=_mi_enlarge_root(info,keyinfo,key,root); + } + else /* error == 1 */ + { + if (mi_getint(root_buff) <= (nod_flag=mi_test_if_nod(root_buff))+3) + { + error=0; + if (nod_flag) + *root=_mi_kpos(nod_flag,root_buff+2+nod_flag); + else + *root=HA_OFFSET_ERROR; + if (_mi_dispose(info,keyinfo,old_root,DFLT_INIT_HITS)) + error= -1; + } + else + error=_mi_write_keypage(info,keyinfo,old_root, + DFLT_INIT_HITS,root_buff); + } + } +err: + my_afree((uchar*) root_buff); + DBUG_PRINT("exit",("Return: %d",error)); + DBUG_RETURN(error); +} /* _mi_ck_real_delete */ + + + /* + ** Remove key below key root + ** Return values: + ** 1 if there are less buffers; In this case anc_buff is not saved + ** 2 if there are more buffers + ** -1 on errors + */ + +static int d_search(register MI_INFO *info, register MI_KEYDEF *keyinfo, + uint comp_flag, uchar *key, uint key_length, + my_off_t page, uchar *anc_buff) +{ + int flag,ret_value,save_flag; + uint length,nod_flag,search_key_length; + my_bool last_key; + uchar *leaf_buff,*keypos; + my_off_t UNINIT_VAR(leaf_page),next_block; + uchar lastkey[HA_MAX_KEY_BUFF]; + DBUG_ENTER("d_search"); + DBUG_DUMP("page",(uchar*) anc_buff,mi_getint(anc_buff)); + + search_key_length= (comp_flag & SEARCH_FIND) ? key_length : USE_WHOLE_KEY; + flag=(*keyinfo->bin_search)(info,keyinfo,anc_buff,key, search_key_length, + comp_flag, &keypos, lastkey, &last_key); + if (flag == MI_FOUND_WRONG_KEY) + { + DBUG_PRINT("error",("Found wrong key")); + DBUG_RETURN(-1); + } + nod_flag=mi_test_if_nod(anc_buff); + + if (!flag && keyinfo->flag & HA_FULLTEXT) + { + uint off; + int subkeys; + + get_key_full_length_rdonly(off, lastkey); + subkeys=ft_sintXkorr(lastkey+off); + DBUG_ASSERT(info->ft1_to_ft2==0 || subkeys >=0); + comp_flag=SEARCH_SAME; + if (subkeys >= 0) + { + /* normal word, one-level tree structure */ + if (info->ft1_to_ft2) + { + /* we're in ft1->ft2 conversion mode. Saving key data */ + if (insert_dynamic(info->ft1_to_ft2, (lastkey+off))) + { + DBUG_PRINT("error",("Out of memory")); + DBUG_RETURN(-1); + } + } + else + { + /* we need exact match only if not in ft1->ft2 conversion mode */ + flag=(*keyinfo->bin_search)(info,keyinfo,anc_buff,key,USE_WHOLE_KEY, + comp_flag, &keypos, lastkey, &last_key); + } + /* fall through to normal delete */ + } + else + { + /* popular word. two-level tree. going down */ + uint tmp_key_length; + my_off_t root; + uchar *kpos=keypos; + + if (!(tmp_key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&kpos,lastkey))) + { + mi_print_error(info->s, HA_ERR_CRASHED); + my_errno= HA_ERR_CRASHED; + DBUG_RETURN(-1); + } + root=_mi_dpos(info,nod_flag,kpos); + if (subkeys == -1) + { + /* the last entry in sub-tree */ + if (_mi_dispose(info, keyinfo, root,DFLT_INIT_HITS)) + DBUG_RETURN(-1); + /* fall through to normal delete */ + } + else + { + keyinfo=&info->s->ft2_keyinfo; + kpos-=keyinfo->keylength+nod_flag; /* we'll modify key entry 'in vivo' */ + get_key_full_length_rdonly(off, key); + key+=off; + ret_value=_mi_ck_real_delete(info, &info->s->ft2_keyinfo, + key, HA_FT_WLEN, &root); + _mi_dpointer(info, kpos+HA_FT_WLEN, root); + subkeys++; + ft_intXstore(kpos, subkeys); + if (!ret_value) + ret_value=_mi_write_keypage(info,keyinfo,page, + DFLT_INIT_HITS,anc_buff); + DBUG_PRINT("exit",("Return: %d",ret_value)); + DBUG_RETURN(ret_value); + } + } + } + leaf_buff=0; + if (nod_flag) + { + leaf_page=_mi_kpos(nod_flag,keypos); + if (!(leaf_buff= (uchar*) my_alloca((uint) keyinfo->block_length+ + HA_MAX_KEY_BUFF*2))) + { + DBUG_PRINT("error",("Couldn't allocate memory")); + my_errno=ENOMEM; + DBUG_PRINT("exit",("Return: %d",-1)); + DBUG_RETURN(-1); + } + if (!_mi_fetch_keypage(info,keyinfo,leaf_page,DFLT_INIT_HITS,leaf_buff,0)) + goto err; + } + + if (flag != 0) + { + if (!nod_flag) + { + DBUG_PRINT("error",("Didn't find key")); + mi_print_error(info->s, HA_ERR_CRASHED); + my_errno=HA_ERR_CRASHED; /* This should never happen */ + goto err; + } + save_flag=0; + ret_value=d_search(info,keyinfo,comp_flag,key,key_length, + leaf_page,leaf_buff); + } + else + { /* Found key */ + uint tmp; + length=mi_getint(anc_buff); + if (!(tmp= remove_key(keyinfo,nod_flag,keypos,lastkey,anc_buff+length, + &next_block))) + goto err; + + length-= tmp; + + mi_putint(anc_buff,length,nod_flag); + if (!nod_flag) + { /* On leaf page */ + if (_mi_write_keypage(info,keyinfo,page,DFLT_INIT_HITS,anc_buff)) + { + DBUG_PRINT("exit",("Return: %d",-1)); + DBUG_RETURN(-1); + } + /* Page will be update later if we return 1 */ + DBUG_RETURN(MY_TEST(length <= (info->quick_mode ? MI_MIN_KEYBLOCK_LENGTH : + (uint) keyinfo->underflow_block_length))); + } + save_flag=1; + ret_value=del(info,keyinfo,key,anc_buff,leaf_page,leaf_buff,keypos, + next_block,lastkey); + } + if (ret_value >0) + { + save_flag=1; + if (ret_value == 1) + ret_value= underflow(info,keyinfo,anc_buff,leaf_page,leaf_buff,keypos); + else + { /* This happens only with packed keys */ + DBUG_PRINT("test",("Enlarging of key when deleting")); + if (!_mi_get_last_key(info,keyinfo,anc_buff,lastkey,keypos,&length)) + goto err; + ret_value=_mi_insert(info,keyinfo,key,anc_buff,keypos,lastkey, + (uchar*) 0,(uchar*) 0,(my_off_t) 0,(my_bool) 0); + } + } + if (ret_value == 0 && mi_getint(anc_buff) > keyinfo->block_length) + { + save_flag=1; + ret_value=_mi_split_page(info,keyinfo,key,anc_buff,lastkey,0) | 2; + } + if (save_flag && ret_value != 1) + ret_value|=_mi_write_keypage(info,keyinfo,page,DFLT_INIT_HITS,anc_buff); + else + { + DBUG_DUMP("page",(uchar*) anc_buff,mi_getint(anc_buff)); + } + my_afree((uchar*) leaf_buff); + DBUG_PRINT("exit",("Return: %d",ret_value)); + DBUG_RETURN(ret_value); + +err: + my_afree((uchar*) leaf_buff); + DBUG_PRINT("exit",("Error: %d",my_errno)); + DBUG_RETURN (-1); +} /* d_search */ + + + /* Remove a key that has a page-reference */ + +static int del(register MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *key, + uchar *anc_buff, my_off_t leaf_page, uchar *leaf_buff, + uchar *keypos, /* Pos to where deleted key was */ + my_off_t next_block, + uchar *ret_key) /* key before keypos in anc_buff */ +{ + int ret_value,length; + uint a_length,nod_flag,tmp; + my_off_t next_page; + uchar keybuff[HA_MAX_KEY_BUFF],*endpos,*next_buff,*key_start, *prev_key; + MYISAM_SHARE *share=info->s; + MI_KEY_PARAM s_temp; + DBUG_ENTER("del"); + DBUG_PRINT("enter",("leaf_page: %lld keypos: %p", leaf_page, + keypos)); + DBUG_DUMP("leaf_buff",(uchar*) leaf_buff,mi_getint(leaf_buff)); + + endpos=leaf_buff+mi_getint(leaf_buff); + if (!(key_start=_mi_get_last_key(info,keyinfo,leaf_buff,keybuff,endpos, + &tmp))) + DBUG_RETURN(-1); + + if ((nod_flag=mi_test_if_nod(leaf_buff))) + { + next_page= _mi_kpos(nod_flag,endpos); + if (!(next_buff= (uchar*) my_alloca((uint) keyinfo->block_length+ + HA_MAX_KEY_BUFF*2))) + DBUG_RETURN(-1); + if (!_mi_fetch_keypage(info,keyinfo,next_page,DFLT_INIT_HITS,next_buff,0)) + ret_value= -1; + else + { + DBUG_DUMP("next_page",(uchar*) next_buff,mi_getint(next_buff)); + if ((ret_value=del(info,keyinfo,key,anc_buff,next_page,next_buff, + keypos,next_block,ret_key)) >0) + { + endpos=leaf_buff+mi_getint(leaf_buff); + if (ret_value == 1) + { + ret_value=underflow(info,keyinfo,leaf_buff,next_page, + next_buff,endpos); + if (ret_value == 0 && mi_getint(leaf_buff) > keyinfo->block_length) + { + ret_value=_mi_split_page(info,keyinfo,key,leaf_buff,ret_key,0) | 2; + } + } + else + { + DBUG_PRINT("test",("Inserting of key when deleting")); + if (!_mi_get_last_key(info,keyinfo,leaf_buff,keybuff,endpos, + &tmp)) + goto err; + ret_value=_mi_insert(info,keyinfo,key,leaf_buff,endpos,keybuff, + (uchar*) 0,(uchar*) 0,(my_off_t) 0,0); + } + } + if (_mi_write_keypage(info,keyinfo,leaf_page,DFLT_INIT_HITS,leaf_buff)) + goto err; + } + my_afree((uchar*) next_buff); + DBUG_RETURN(ret_value); + } + + /* Remove last key from leaf page */ + + mi_putint(leaf_buff,key_start-leaf_buff,nod_flag); + if (_mi_write_keypage(info,keyinfo,leaf_page,DFLT_INIT_HITS,leaf_buff)) + goto err; + + /* Place last key in ancestor page on deleted key position */ + + a_length=mi_getint(anc_buff); + endpos=anc_buff+a_length; + if (keypos != anc_buff+2+share->base.key_reflength && + !_mi_get_last_key(info,keyinfo,anc_buff,ret_key,keypos,&tmp)) + goto err; + prev_key=(keypos == anc_buff+2+share->base.key_reflength ? + 0 : ret_key); + length=(*keyinfo->pack_key)(keyinfo,share->base.key_reflength, + keypos == endpos ? (uchar*) 0 : keypos, + prev_key, prev_key, + keybuff,&s_temp); + if (length > 0) + bmove_upp((uchar*) endpos+length,(uchar*) endpos,(uint) (endpos-keypos)); + else + bmove(keypos,keypos-length, (int) (endpos-keypos)+length); + (*keyinfo->store_key)(keyinfo,keypos,&s_temp); + /* Save pointer to next leaf */ + if (!(*keyinfo->get_key)(keyinfo,share->base.key_reflength,&keypos,ret_key)) + goto err; + _mi_kpointer(info,keypos - share->base.key_reflength,next_block); + mi_putint(anc_buff,a_length+length,share->base.key_reflength); + + DBUG_RETURN( mi_getint(leaf_buff) <= + (info->quick_mode ? MI_MIN_KEYBLOCK_LENGTH : + (uint) keyinfo->underflow_block_length)); +err: + DBUG_RETURN(-1); +} /* del */ + + + /* Balances adjacent pages if underflow occours */ + +static int underflow(register MI_INFO *info, register MI_KEYDEF *keyinfo, + uchar *anc_buff, + my_off_t leaf_page,/* Ancestor page and underflow page */ + uchar *leaf_buff, + uchar *keypos) /* Position to pos after key */ +{ + int t_length; + uint length,anc_length,buff_length,leaf_length,p_length,s_length,nod_flag, + key_reflength,key_length; + my_off_t next_page; + uchar anc_key[HA_MAX_KEY_BUFF],leaf_key[HA_MAX_KEY_BUFF], + *buff,*endpos,*next_keypos,*anc_pos,*half_pos,*temp_pos,*prev_key, + *after_key; + MI_KEY_PARAM s_temp; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("underflow"); + DBUG_PRINT("enter",("leaf_page: %lld keypos: %p",leaf_page, + keypos)); + DBUG_DUMP("anc_buff",(uchar*) anc_buff,mi_getint(anc_buff)); + DBUG_DUMP("leaf_buff",(uchar*) leaf_buff,mi_getint(leaf_buff)); + + buff=info->buff; + info->buff_used=1; + next_keypos=keypos; + nod_flag=mi_test_if_nod(leaf_buff); + p_length=nod_flag+2; + anc_length=mi_getint(anc_buff); + leaf_length=mi_getint(leaf_buff); + key_reflength=share->base.key_reflength; + if (info->s->keyinfo+info->lastinx == keyinfo) + info->page_changed=1; + + if ((keypos < anc_buff+anc_length && (info->state->records & 1)) || + keypos == anc_buff+2+key_reflength) + { /* Use page right of anc-page */ + DBUG_PRINT("test",("use right page")); + + if (keyinfo->flag & HA_BINARY_PACK_KEY) + { + if (!(next_keypos=_mi_get_key(info, keyinfo, + anc_buff, buff, keypos, &length))) + goto err; + } + else + { + /* Got to end of found key */ + buff[0]=buff[1]=0; /* Avoid length error check if packed key */ + if (!(*keyinfo->get_key)(keyinfo,key_reflength,&next_keypos, + buff)) + goto err; + } + next_page= _mi_kpos(key_reflength,next_keypos); + if (!_mi_fetch_keypage(info,keyinfo,next_page,DFLT_INIT_HITS,buff,0)) + goto err; + buff_length=mi_getint(buff); + DBUG_DUMP("next",(uchar*) buff,buff_length); + + /* find keys to make a big key-page */ + bmove((uchar*) next_keypos-key_reflength,(uchar*) buff+2, + key_reflength); + if (!_mi_get_last_key(info,keyinfo,anc_buff,anc_key,next_keypos,&length) + || !_mi_get_last_key(info,keyinfo,leaf_buff,leaf_key, + leaf_buff+leaf_length,&length)) + goto err; + + /* merge pages and put parting key from anc_buff between */ + prev_key=(leaf_length == p_length ? (uchar*) 0 : leaf_key); + t_length=(*keyinfo->pack_key)(keyinfo,nod_flag,buff+p_length, + prev_key, prev_key, + anc_key, &s_temp); + length=buff_length-p_length; + endpos=buff+length+leaf_length+t_length; + /* buff will always be larger than before !*/ + bmove_upp((uchar*) endpos, (uchar*) buff+buff_length,length); + memcpy((uchar*) buff, (uchar*) leaf_buff,(size_t) leaf_length); + (*keyinfo->store_key)(keyinfo,buff+leaf_length,&s_temp); + buff_length=(uint) (endpos-buff); + mi_putint(buff,buff_length,nod_flag); + + /* remove key from anc_buff */ + + if (!(s_length=remove_key(keyinfo,key_reflength,keypos,anc_key, + anc_buff+anc_length,(my_off_t *) 0))) + goto err; + + anc_length-=s_length; + mi_putint(anc_buff,anc_length,key_reflength); + + if (buff_length <= keyinfo->block_length) + { /* Keys in one page */ + memcpy((uchar*) leaf_buff,(uchar*) buff,(size_t) buff_length); + if (_mi_dispose(info,keyinfo,next_page,DFLT_INIT_HITS)) + goto err; + } + else + { /* Page is full */ + endpos=anc_buff+anc_length; + DBUG_PRINT("test",("anc_buff: %p endpos: %p", + anc_buff, endpos)); + if (keypos != anc_buff+2+key_reflength && + !_mi_get_last_key(info,keyinfo,anc_buff,anc_key,keypos,&length)) + goto err; + if (!(half_pos=_mi_find_half_pos(nod_flag, keyinfo, buff, leaf_key, + &key_length, &after_key))) + goto err; + length=(uint) (half_pos-buff); + memcpy((uchar*) leaf_buff,(uchar*) buff,(size_t) length); + mi_putint(leaf_buff,length,nod_flag); + + /* Correct new keypointer to leaf_page */ + half_pos=after_key; + _mi_kpointer(info,leaf_key+key_length,next_page); + /* Save key in anc_buff */ + prev_key=(keypos == anc_buff+2+key_reflength ? (uchar*) 0 : anc_key), + t_length=(*keyinfo->pack_key)(keyinfo,key_reflength, + (keypos == endpos ? (uchar*) 0 : + keypos), + prev_key, prev_key, + leaf_key, &s_temp); + if (t_length >= 0) + bmove_upp((uchar*) endpos+t_length,(uchar*) endpos, + (uint) (endpos-keypos)); + else + bmove(keypos,keypos-t_length,(uint) (endpos-keypos)+t_length); + (*keyinfo->store_key)(keyinfo,keypos,&s_temp); + mi_putint(anc_buff,(anc_length+=t_length),key_reflength); + + /* Store key first in new page */ + if (nod_flag) + bmove((uchar*) buff+2,(uchar*) half_pos-nod_flag,(size_t) nod_flag); + if (!(*keyinfo->get_key)(keyinfo,nod_flag,&half_pos,leaf_key)) + goto err; + t_length=(int) (*keyinfo->pack_key)(keyinfo, nod_flag, (uchar*) 0, + (uchar*) 0, (uchar *) 0, + leaf_key, &s_temp); + /* t_length will always be > 0 for a new page !*/ + length=(uint) ((buff+mi_getint(buff))-half_pos); + bmove((uchar*) buff+p_length+t_length,(uchar*) half_pos,(size_t) length); + (*keyinfo->store_key)(keyinfo,buff+p_length,&s_temp); + mi_putint(buff,length+t_length+p_length,nod_flag); + + if (_mi_write_keypage(info,keyinfo,next_page,DFLT_INIT_HITS,buff)) + goto err; + } + if (_mi_write_keypage(info,keyinfo,leaf_page,DFLT_INIT_HITS,leaf_buff)) + goto err; + DBUG_RETURN(anc_length <= ((info->quick_mode ? MI_MIN_BLOCK_LENGTH : + (uint) keyinfo->underflow_block_length))); + } + + DBUG_PRINT("test",("use left page")); + + keypos=_mi_get_last_key(info,keyinfo,anc_buff,anc_key,keypos,&length); + if (!keypos) + goto err; + next_page= _mi_kpos(key_reflength,keypos); + if (!_mi_fetch_keypage(info,keyinfo,next_page,DFLT_INIT_HITS,buff,0)) + goto err; + buff_length=mi_getint(buff); + endpos=buff+buff_length; + DBUG_DUMP("prev",(uchar*) buff,buff_length); + + /* find keys to make a big key-page */ + bmove((uchar*) next_keypos - key_reflength,(uchar*) leaf_buff+2, + key_reflength); + next_keypos=keypos; + if (!(*keyinfo->get_key)(keyinfo,key_reflength,&next_keypos, + anc_key)) + goto err; + if (!_mi_get_last_key(info,keyinfo,buff,leaf_key,endpos,&length)) + goto err; + + /* merge pages and put parting key from anc_buff between */ + prev_key=(leaf_length == p_length ? (uchar*) 0 : leaf_key); + t_length=(*keyinfo->pack_key)(keyinfo,nod_flag, + (leaf_length == p_length ? + (uchar*) 0 : leaf_buff+p_length), + prev_key, prev_key, + anc_key, &s_temp); + if (t_length >= 0) + bmove((uchar*) endpos+t_length,(uchar*) leaf_buff+p_length, + (size_t) (leaf_length-p_length)); + else /* We gained space */ + bmove((uchar*) endpos,(uchar*) leaf_buff+((int) p_length-t_length), + (size_t) (leaf_length-p_length+t_length)); + + (*keyinfo->store_key)(keyinfo,endpos,&s_temp); + buff_length=buff_length+leaf_length-p_length+t_length; + mi_putint(buff,buff_length,nod_flag); + + /* remove key from anc_buff */ + if (!(s_length= remove_key(keyinfo,key_reflength,keypos,anc_key, + anc_buff+anc_length,(my_off_t *) 0))) + goto err; + + anc_length-=s_length; + mi_putint(anc_buff,anc_length,key_reflength); + + if (buff_length <= keyinfo->block_length) + { /* Keys in one page */ + if (_mi_dispose(info,keyinfo,leaf_page,DFLT_INIT_HITS)) + goto err; + } + else + { /* Page is full */ + if (keypos == anc_buff+2+key_reflength) + anc_pos=0; /* First key */ + else if (!_mi_get_last_key(info,keyinfo,anc_buff,anc_pos=anc_key,keypos, + &length)) + goto err; + endpos=_mi_find_half_pos(nod_flag,keyinfo,buff,leaf_key, + &key_length, &half_pos); + if (!endpos) + goto err; + _mi_kpointer(info,leaf_key+key_length,leaf_page); + /* Save key in anc_buff */ + DBUG_DUMP("anc_buff",(uchar*) anc_buff,anc_length); + DBUG_DUMP("key_to_anc",(uchar*) leaf_key,key_length); + + temp_pos=anc_buff+anc_length; + t_length=(*keyinfo->pack_key)(keyinfo,key_reflength, + keypos == temp_pos ? (uchar*) 0 + : keypos, + anc_pos, anc_pos, + leaf_key,&s_temp); + if (t_length > 0) + bmove_upp((uchar*) temp_pos+t_length,(uchar*) temp_pos, + (uint) (temp_pos-keypos)); + else + bmove(keypos,keypos-t_length,(uint) (temp_pos-keypos)+t_length); + (*keyinfo->store_key)(keyinfo,keypos,&s_temp); + mi_putint(anc_buff,(anc_length+=t_length),key_reflength); + + /* Store first key on new page */ + if (nod_flag) + bmove((uchar*) leaf_buff+2,(uchar*) half_pos-nod_flag,(size_t) nod_flag); + if (!(length=(*keyinfo->get_key)(keyinfo,nod_flag,&half_pos,leaf_key))) + goto err; + DBUG_DUMP("key_to_leaf",(uchar*) leaf_key,length); + t_length=(*keyinfo->pack_key)(keyinfo,nod_flag, (uchar*) 0, + (uchar*) 0, (uchar*) 0, leaf_key, &s_temp); + length=(uint) ((buff+buff_length)-half_pos); + DBUG_PRINT("info",("t_length: %d length: %d",t_length,(int) length)); + bmove((uchar*) leaf_buff+p_length+t_length,(uchar*) half_pos, + (size_t) length); + (*keyinfo->store_key)(keyinfo,leaf_buff+p_length,&s_temp); + mi_putint(leaf_buff,length+t_length+p_length,nod_flag); + if (_mi_write_keypage(info,keyinfo,leaf_page,DFLT_INIT_HITS,leaf_buff)) + goto err; + mi_putint(buff,endpos-buff,nod_flag); + } + if (_mi_write_keypage(info,keyinfo,next_page,DFLT_INIT_HITS,buff)) + goto err; + DBUG_RETURN(anc_length <= (uint) keyinfo->block_length/2); + +err: + DBUG_RETURN(-1); +} /* underflow */ + + + /* + remove a key from packed buffert + The current code doesn't handle the case that the next key may be + packed better against the previous key if there is a case difference + returns how many chars was removed or 0 on error + */ + +#if defined(_MSC_VER) && defined(_M_X64) && _MSC_VER >= 1930 +#pragma optimize("g", off) +#endif + +static uint remove_key(MI_KEYDEF *keyinfo, uint nod_flag, + uchar *keypos, /* Where key starts */ + uchar *lastkey, /* key to be removed */ + uchar *page_end, /* End of page */ + my_off_t *next_block) /* ptr to next block */ +{ + int s_length; + uchar *start; + DBUG_ENTER("remove_key"); + DBUG_PRINT("enter",("keypos: %p page_end: %p",keypos, page_end)); + + start=keypos; + if (!(keyinfo->flag & + (HA_PACK_KEY | HA_SPACE_PACK_USED | HA_VAR_LENGTH_KEY | + HA_BINARY_PACK_KEY))) + { + s_length=(int) (keyinfo->keylength+nod_flag); + if (next_block && nod_flag) + *next_block= _mi_kpos(nod_flag,keypos+s_length); + } + else + { /* Let keypos point at next key */ + /* Calculate length of key */ + if (!(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,lastkey)) + DBUG_RETURN(0); /* Error */ + + if (next_block && nod_flag) + *next_block= _mi_kpos(nod_flag,keypos); + s_length=(int) (keypos-start); + if (keypos != page_end) + { + if (keyinfo->flag & HA_BINARY_PACK_KEY) + { + uchar *old_key=start; + uint next_length,prev_length,prev_pack_length; + get_key_length(next_length,keypos); + get_key_pack_length(prev_length,prev_pack_length,old_key); + if (next_length > prev_length) + { + /* We have to copy data from the current key to the next key */ + bmove_upp(keypos, (lastkey+next_length), + (next_length-prev_length)); + keypos-=(next_length-prev_length)+prev_pack_length; + store_key_length(keypos,prev_length); + s_length=(int) (keypos-start); + } + } + else + { + /* Check if a variable length first key part */ + if ((keyinfo->seg->flag & HA_PACK_KEY) && *keypos & 128) + { + /* Next key is packed against the current one */ + uint next_length,prev_length,prev_pack_length,lastkey_length, + rest_length; + if (keyinfo->seg[0].length >= 127) + { + if (!(prev_length=mi_uint2korr(start) & 32767)) + goto end; + next_length=mi_uint2korr(keypos) & 32767; + keypos+=2; + prev_pack_length=2; + } + else + { + if (!(prev_length= *start & 127)) + goto end; /* Same key as previous*/ + next_length= *keypos & 127; + keypos++; + prev_pack_length=1; + } + if (!(*start & 128)) + prev_length=0; /* prev key not packed */ + if (keyinfo->seg[0].flag & HA_NULL_PART) + lastkey++; /* Skip null marker */ + get_key_length(lastkey_length,lastkey); + if (!next_length) /* Same key after */ + { + next_length=lastkey_length; + rest_length=0; + } + else + get_key_length(rest_length,keypos); + + if (next_length >= prev_length) + { /* Key after is based on deleted key */ + uint pack_length,tmp; + bmove_upp(keypos, (lastkey+next_length), + tmp=(next_length-prev_length)); + rest_length+=tmp; + pack_length= prev_length ? get_pack_length(rest_length): 0; + keypos-=tmp+pack_length+prev_pack_length; + s_length=(int) (keypos-start); + if (prev_length) /* Pack against prev key */ + { + *keypos++= start[0]; + if (prev_pack_length == 2) + *keypos++= start[1]; + store_key_length(keypos,rest_length); + } + else + { + /* Next key is not packed anymore */ + if (keyinfo->seg[0].flag & HA_NULL_PART) + { + rest_length++; /* Mark not null */ + } + if (prev_pack_length == 2) + { + mi_int2store(keypos,rest_length); + } + else + *keypos= rest_length; + } + } + } + } + } + } + end: + bmove((uchar*) start,(uchar*) start+s_length, + (uint) (page_end-start-s_length)); + DBUG_RETURN((uint) s_length); +} /* remove_key */ + +#if defined(_MSC_VER) && defined(_M_X64) && _MSC_VER >= 1930 +#pragma optimize("",on) +#endif diff --git a/storage/myisam/mi_delete_all.c b/storage/myisam/mi_delete_all.c new file mode 100644 index 00000000..4bfe0e8d --- /dev/null +++ b/storage/myisam/mi_delete_all.c @@ -0,0 +1,79 @@ +/* + Copyright (c) 2000, 2011, Oracle and/or its affiliates + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Remove all rows from a MyISAM table */ +/* This clears the status information and truncates files */ + +#include "myisamdef.h" + +int mi_delete_all_rows(MI_INFO *info) +{ + uint i; + MYISAM_SHARE *share=info->s; + MI_STATE_INFO *state=&share->state; + DBUG_ENTER("mi_delete_all_rows"); + + if (share->options & HA_OPTION_READ_ONLY_DATA) + { + DBUG_RETURN(my_errno=EACCES); + } + if (_mi_readinfo(info,F_WRLCK,1)) + DBUG_RETURN(my_errno); + if (_mi_mark_file_changed(info)) + goto err; + + info->state->records=info->state->del=state->split=0; + state->dellink = HA_OFFSET_ERROR; + state->sortkey= (ushort) ~0; + info->state->key_file_length=share->base.keystart; + info->state->data_file_length=0; + info->state->empty=info->state->key_empty=0; + info->state->checksum=0; + + for (i=share->base.max_key_block_length/MI_MIN_KEY_BLOCK_LENGTH ; i-- ; ) + state->key_del[i]= HA_OFFSET_ERROR; + for (i=0 ; i < share->base.keys ; i++) + state->key_root[i]= HA_OFFSET_ERROR; + + myisam_log_command(MI_LOG_DELETE_ALL,info,(uchar*) 0,0,0); + /* + If we are using delayed keys or if the user has done changes to the tables + since it was locked then there may be key blocks in the key cache + */ + flush_key_blocks(share->key_cache, share->kfile, &share->dirty_part_map, + FLUSH_IGNORE_CHANGED); +#ifdef HAVE_MMAP + if (share->file_map) + mi_munmap_file(info); +#endif + if (mysql_file_chsize(info->dfile, 0, 0, MYF(MY_WME)) || + mysql_file_chsize(share->kfile, share->base.keystart, 0, MYF(MY_WME))) + goto err; + + if (info->opt_flag & WRITE_CACHE_USED) + reinit_io_cache(&info->rec_cache, WRITE_CACHE, 0, 1, 1); + + (void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE); + DBUG_RETURN(0); + +err: + { + int save_errno=my_errno; + (void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE); + info->update|=HA_STATE_WRITTEN; /* Buffer changed */ + DBUG_RETURN(my_errno=save_errno); + } +} /* mi_delete */ diff --git a/storage/myisam/mi_delete_table.c b/storage/myisam/mi_delete_table.c new file mode 100644 index 00000000..d318b447 --- /dev/null +++ b/storage/myisam/mi_delete_table.c @@ -0,0 +1,50 @@ +/* + Copyright (c) 2000, 2010, Oracle and/or its affiliates + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* + deletes a table +*/ + +#include "fulltext.h" + +#ifndef HAVE_PSI_INTERFACE +#define PSI_file_key int +#define mi_key_file_kfile 0 +#define mi_key_file_dfile 0 +#endif + +int mi_delete_table(const char *name) +{ + int error= 0; + DBUG_ENTER("mi_delete_table"); + +#ifdef EXTRA_DEBUG + check_table_is_closed(name,"delete"); +#endif + + if (mysql_file_delete_with_symlink(mi_key_file_kfile, name, MI_NAME_IEXT, + MYF(MY_WME))) + error= my_errno; + if (mysql_file_delete_with_symlink(mi_key_file_dfile, name, MI_NAME_DEXT, + MYF(MY_WME))) + error= my_errno; + + // optionally present: + mysql_file_delete_with_symlink(mi_key_file_dfile, name, ".OLD", MYF(0)); + mysql_file_delete_with_symlink(mi_key_file_dfile, name, ".TMD", MYF(0)); + + DBUG_RETURN(error); +} diff --git a/storage/myisam/mi_dynrec.c b/storage/myisam/mi_dynrec.c new file mode 100644 index 00000000..09c10040 --- /dev/null +++ b/storage/myisam/mi_dynrec.c @@ -0,0 +1,2030 @@ +/* + Copyright (c) 2000, 2011, Oracle and/or its affiliates + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* + Functions to handle space-packed-records and blobs + + A row may be stored in one or more linked blocks. + The block size is between MI_MIN_BLOCK_LENGTH and MI_MAX_BLOCK_LENGTH. + Each block is aligned on MI_DYN_ALIGN_SIZE. + The reson for the max block size is to not have too many different types + of blocks. For the differnet block types, look at _mi_get_block_info() +*/ + +#include "myisamdef.h" + +/* Enough for comparing if number is zero */ +static char zero_string[]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; + +static int write_dynamic_record(MI_INFO *info,const uchar *record, + ulong reclength); +static int _mi_find_writepos(MI_INFO *info,ulong reclength,my_off_t *filepos, + ulong *length); +static int update_dynamic_record(MI_INFO *info,my_off_t filepos,uchar *record, + ulong reclength); +static int delete_dynamic_record(MI_INFO *info,my_off_t filepos, + uint second_read); +static int _mi_cmp_buffer(File file, const uchar *buff, my_off_t filepos, + uint length); + +/* Play it safe; We have a small stack when using threads */ +#undef my_alloca +#undef my_afree +#define my_alloca(A) my_malloc(PSI_NOT_INSTRUMENTED, (A),MYF(0)) +#define my_afree(A) my_free((A)) + + /* Interface function from MI_INFO */ + +#ifdef HAVE_MMAP + +/* + Create mmaped area for MyISAM handler + + SYNOPSIS + mi_dynmap_file() + info MyISAM handler + + RETURN + 0 ok + 1 error. +*/ + +my_bool mi_dynmap_file(MI_INFO *info, my_off_t size) +{ + DBUG_ENTER("mi_dynmap_file"); + if (size == 0 || size > (my_off_t) (~((size_t) 0))) + { + if (size) + DBUG_PRINT("warning", ("File is too large for mmap")); + else + DBUG_PRINT("warning", ("Do not mmap zero-length")); + DBUG_RETURN(1); + } + /* + I wonder if it is good to use MAP_NORESERVE. From the Linux man page: + MAP_NORESERVE + Do not reserve swap space for this mapping. When swap space is + reserved, one has the guarantee that it is possible to modify the + mapping. When swap space is not reserved one might get SIGSEGV + upon a write if no physical memory is available. + */ + info->s->file_map= (uchar*) + my_mmap(0, (size_t) size, + info->s->mode==O_RDONLY ? PROT_READ : + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_NORESERVE, + info->dfile, 0L); + if (info->s->file_map == (uchar*) MAP_FAILED) + { + info->s->file_map= NULL; + DBUG_RETURN(1); + } +#if defined(HAVE_MADVISE) + madvise((char*) info->s->file_map, size, MADV_RANDOM); +#endif + info->s->mmaped_length= (size_t) size; + info->s->file_read= mi_mmap_pread; + info->s->file_write= mi_mmap_pwrite; + DBUG_RETURN(0); +} + + +/* + Destroy mmaped area for MyISAM handler + + SYNOPSIS + mi_munmap_file() + info MyISAM handler + + RETURN + 0 ok + !0 error. +*/ + +int mi_munmap_file(MI_INFO *info) +{ + int ret; + DBUG_ENTER("mi_unmap_file"); + if ((ret= my_munmap((void*) info->s->file_map, info->s->mmaped_length))) + DBUG_RETURN(ret); + info->s->file_read= mi_nommap_pread; + info->s->file_write= mi_nommap_pwrite; + info->s->file_map= 0; + info->s->mmaped_length= 0; + DBUG_RETURN(0); +} + + +/* + Resize mmaped area for MyISAM handler + + SYNOPSIS + mi_remap_file() + info MyISAM handler + + RETURN +*/ + +void mi_remap_file(MI_INFO *info, my_off_t size) +{ + if (info->s->file_map) + { + mi_munmap_file(info); + mi_dynmap_file(info, size); + } +} +#endif + + +/* + Read bytes from MySAM handler, using mmap or pread + + SYNOPSIS + mi_mmap_pread() + info MyISAM handler + Buffer Input buffer + Count Count of bytes for read + offset Start position + MyFlags + + RETURN + 0 ok +*/ + +size_t mi_mmap_pread(MI_INFO *info, uchar *Buffer, + size_t Count, my_off_t offset, myf MyFlags) +{ + DBUG_PRINT("info", ("mi_read with mmap %d\n", info->dfile)); + if (info->s->concurrent_insert) + mysql_rwlock_rdlock(&info->s->mmap_lock); + + /* + The following test may fail in the following cases: + - We failed to remap a memory area (fragmented memory?) + - This thread has done some writes, but not yet extended the + memory mapped area. + */ + + if (info->s->mmaped_length >= offset + Count) + { + memcpy(Buffer, info->s->file_map + offset, Count); + if (info->s->concurrent_insert) + mysql_rwlock_unlock(&info->s->mmap_lock); + return 0; + } + else + { + if (info->s->concurrent_insert) + mysql_rwlock_unlock(&info->s->mmap_lock); + return mysql_file_pread(info->dfile, Buffer, Count, offset, MyFlags); + } +} + + + /* wrapper for mysql_file_pread in case if mmap isn't used */ + +size_t mi_nommap_pread(MI_INFO *info, uchar *Buffer, + size_t Count, my_off_t offset, myf MyFlags) +{ + return mysql_file_pread(info->dfile, Buffer, Count, offset, MyFlags); +} + + +/* + Write bytes to MySAM handler, using mmap or pwrite + + SYNOPSIS + mi_mmap_pwrite() + info MyISAM handler + Buffer Output buffer + Count Count of bytes for write + offset Start position + MyFlags + + RETURN + 0 ok + !=0 error. In this case return error from pwrite +*/ + +size_t mi_mmap_pwrite(MI_INFO *info, const uchar *Buffer, + size_t Count, my_off_t offset, myf MyFlags) +{ + DBUG_PRINT("info", ("mi_write with mmap %d\n", info->dfile)); + if (info->s->concurrent_insert) + mysql_rwlock_rdlock(&info->s->mmap_lock); + + /* + The following test may fail in the following cases: + - We failed to remap a memory area (fragmented memory?) + - This thread has done some writes, but not yet extended the + memory mapped area. + */ + + if (info->s->mmaped_length >= offset + Count) + { + memcpy(info->s->file_map + offset, Buffer, Count); + if (info->s->concurrent_insert) + mysql_rwlock_unlock(&info->s->mmap_lock); + return 0; + } + else + { + info->s->nonmmaped_inserts++; + if (info->s->concurrent_insert) + mysql_rwlock_unlock(&info->s->mmap_lock); + return mysql_file_pwrite(info->dfile, Buffer, Count, offset, MyFlags); + } + +} + + + /* wrapper for mysql_file_pwrite in case if mmap isn't used */ + +size_t mi_nommap_pwrite(MI_INFO *info, const uchar *Buffer, + size_t Count, my_off_t offset, myf MyFlags) +{ + return mysql_file_pwrite(info->dfile, Buffer, Count, offset, MyFlags); +} + + +int _mi_write_dynamic_record(MI_INFO *info, const uchar *record) +{ + ulong reclength=_mi_rec_pack(info,info->rec_buff,record); + return (write_dynamic_record(info,info->rec_buff,reclength)); +} + +int _mi_update_dynamic_record(MI_INFO *info, my_off_t pos, const uchar *record) +{ + uint length=_mi_rec_pack(info,info->rec_buff,record); + return (update_dynamic_record(info,pos,info->rec_buff,length)); +} + +int _mi_write_blob_record(MI_INFO *info, const uchar *record) +{ + uchar *rec_buff; + int error; + ulong reclength,reclength2,extra; + + extra= (ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+ + MI_DYN_DELETE_BLOCK_HEADER+1); + reclength= (info->s->base.pack_reclength + + _mi_calc_total_blob_length(info,record)+ extra); + if (!(rec_buff=(uchar*) my_alloca(reclength))) + { + my_errno= HA_ERR_OUT_OF_MEM; /* purecov: inspected */ + return(-1); + } + reclength2= _mi_rec_pack(info,rec_buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER), + record); + DBUG_PRINT("info",("reclength: %lu reclength2: %lu", + reclength, reclength2)); + DBUG_ASSERT(reclength2 <= reclength); + error=write_dynamic_record(info,rec_buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER), + reclength2); + my_afree(rec_buff); + return(error); +} + + +int _mi_update_blob_record(MI_INFO *info, my_off_t pos, const uchar *record) +{ + uchar *rec_buff; + int error; + ulong reclength,extra; + + extra= (ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+ + MI_DYN_DELETE_BLOCK_HEADER); + reclength= (info->s->base.pack_reclength+ + _mi_calc_total_blob_length(info,record)+ extra); + if (!(rec_buff=(uchar*) my_alloca(reclength))) + { + my_errno= HA_ERR_OUT_OF_MEM; /* purecov: inspected */ + return(-1); + } + reclength=_mi_rec_pack(info,rec_buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER), + record); + error=update_dynamic_record(info,pos, + rec_buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER), + reclength); + my_afree(rec_buff); + return(error); +} + + +int _mi_delete_dynamic_record(MI_INFO *info) +{ + return delete_dynamic_record(info,info->lastpos,0); +} + + + /* Write record to data-file */ + +static int write_dynamic_record(MI_INFO *info, const uchar *record, + ulong reclength) +{ + int flag; + ulong length; + my_off_t filepos; + DBUG_ENTER("write_dynamic_record"); + + flag=0; + + /* + Check if we have enough room for the new record. + First we do simplified check to make usual case faster. + Then we do more precise check for the space left. + Though it still is not absolutely precise, as + we always use MI_MAX_DYN_BLOCK_HEADER while it can be + less in the most of the cases. + */ + + if (unlikely(info->s->base.max_data_file_length - + info->state->data_file_length < + reclength + MI_MAX_DYN_BLOCK_HEADER)) + { + if (info->s->base.max_data_file_length - info->state->data_file_length + + info->state->empty - info->state->del * MI_MAX_DYN_BLOCK_HEADER < + reclength + MI_MAX_DYN_BLOCK_HEADER) + { + my_errno=HA_ERR_RECORD_FILE_FULL; + DBUG_RETURN(1); + } + } + + do + { + if (_mi_find_writepos(info,reclength,&filepos,&length)) + goto err; + if (_mi_write_part_record(info,filepos,length, + (info->append_insert_at_end ? + HA_OFFSET_ERROR : info->s->state.dellink), + (uchar**) &record,&reclength,&flag)) + goto err; + } while (reclength); + + DBUG_RETURN(0); +err: + DBUG_RETURN(1); +} + + + /* Get a block for data ; The given data-area must be used !! */ + +static int _mi_find_writepos(MI_INFO *info, + ulong reclength, /* record length */ + my_off_t *filepos, /* Return file pos */ + ulong *length) /* length of block at filepos */ +{ + MI_BLOCK_INFO block_info; + ulong tmp; + DBUG_ENTER("_mi_find_writepos"); + + if (info->s->state.dellink != HA_OFFSET_ERROR && + !info->append_insert_at_end) + { + /* Deleted blocks exists; Get last used block */ + *filepos=info->s->state.dellink; + block_info.second_read=0; + info->rec_cache.seek_not_done=1; + if (!(_mi_get_block_info(&block_info,info->dfile,info->s->state.dellink) & + BLOCK_DELETED)) + { + DBUG_PRINT("error",("Delete link crashed")); + my_errno=HA_ERR_WRONG_IN_RECORD; + DBUG_RETURN(-1); + } + info->s->state.dellink=block_info.next_filepos; + info->state->del--; + info->state->empty-= block_info.block_len; + *length= block_info.block_len; + } + else + { + /* No deleted blocks; Allocate a new block */ + *filepos=info->state->data_file_length; + if ((tmp= reclength + 3 + MY_TEST(reclength >= (65520 - 3))) < + info->s->base.min_block_length) + tmp= info->s->base.min_block_length; + else + tmp= ((tmp+MI_DYN_ALIGN_SIZE-1) & + (~ (ulong) (MI_DYN_ALIGN_SIZE-1))); + if (info->state->data_file_length > + (info->s->base.max_data_file_length - tmp)) + { + my_errno=HA_ERR_RECORD_FILE_FULL; + DBUG_RETURN(-1); + } + if (tmp > MI_MAX_BLOCK_LENGTH) + tmp=MI_MAX_BLOCK_LENGTH; + *length= tmp; + info->state->data_file_length+= tmp; + info->s->state.split++; + info->update|=HA_STATE_WRITE_AT_END; + } + DBUG_RETURN(0); +} /* _mi_find_writepos */ + + + +/* + Unlink a deleted block from the deleted list. + This block will be combined with the preceding or next block to form + a big block. +*/ + +static my_bool unlink_deleted_block(MI_INFO *info, MI_BLOCK_INFO *block_info) +{ + DBUG_ENTER("unlink_deleted_block"); + if (block_info->filepos == info->s->state.dellink) + { + /* First deleted block; We can just use this ! */ + info->s->state.dellink=block_info->next_filepos; + } + else + { + MI_BLOCK_INFO tmp; + tmp.second_read=0; + /* Unlink block from the previous block */ + if (!(_mi_get_block_info(&tmp,info->dfile,block_info->prev_filepos) + & BLOCK_DELETED)) + DBUG_RETURN(1); /* Something is wrong */ + mi_sizestore(tmp.header+4,block_info->next_filepos); + if (info->s->file_write(info, tmp.header+4,8, + block_info->prev_filepos+4, MYF(MY_NABP))) + DBUG_RETURN(1); + /* Unlink block from next block */ + if (block_info->next_filepos != HA_OFFSET_ERROR) + { + if (!(_mi_get_block_info(&tmp,info->dfile,block_info->next_filepos) + & BLOCK_DELETED)) + DBUG_RETURN(1); /* Something is wrong */ + mi_sizestore(tmp.header+12,block_info->prev_filepos); + if (info->s->file_write(info, tmp.header+12,8, + block_info->next_filepos+12, + MYF(MY_NABP))) + DBUG_RETURN(1); + } + } + /* We now have one less deleted block */ + info->state->del--; + info->state->empty-= block_info->block_len; + info->s->state.split--; + + /* + If this was a block that we where accessing through table scan + (mi_rrnd() or mi_scan(), then ensure that we skip over this block + when doing next mi_rrnd() or mi_scan(). + */ + if (info->nextpos == block_info->filepos) + info->nextpos+=block_info->block_len; + DBUG_RETURN(0); +} + + +/* + Add a backward link to delete block + + SYNOPSIS + update_backward_delete_link() + info MyISAM handler + delete_block Position to delete block to update. + If this is 'HA_OFFSET_ERROR', nothing will be done + filepos Position to block that 'delete_block' should point to + + RETURN + 0 ok + 1 error. In this case my_error is set. +*/ + +static int update_backward_delete_link(MI_INFO *info, my_off_t delete_block, + my_off_t filepos) +{ + MI_BLOCK_INFO block_info; + DBUG_ENTER("update_backward_delete_link"); + + if (delete_block != HA_OFFSET_ERROR) + { + block_info.second_read=0; + if (_mi_get_block_info(&block_info,info->dfile,delete_block) + & BLOCK_DELETED) + { + uchar buff[8]; + mi_sizestore(buff,filepos); + if (info->s->file_write(info,buff, 8, delete_block+12, MYF(MY_NABP))) + DBUG_RETURN(1); /* Error on write */ + } + else + { + my_errno=HA_ERR_WRONG_IN_RECORD; + DBUG_RETURN(1); /* Wrong delete link */ + } + } + DBUG_RETURN(0); +} + + /* Delete datarecord from database */ + /* info->rec_cache.seek_not_done is updated in cmp_record */ + +static int delete_dynamic_record(MI_INFO *info, my_off_t filepos, + uint second_read) +{ + uint length,b_type; + MI_BLOCK_INFO block_info,del_block; + int error; + my_bool remove_next_block; + DBUG_ENTER("delete_dynamic_record"); + + /* First add a link from the last block to the new one */ + error= update_backward_delete_link(info, info->s->state.dellink, filepos); + + block_info.second_read=second_read; + do + { + /* Remove block at 'filepos' */ + if ((b_type=_mi_get_block_info(&block_info,info->dfile,filepos)) + & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR | + BLOCK_FATAL_ERROR) || + (length=(uint) (block_info.filepos-filepos) +block_info.block_len) < + MI_MIN_BLOCK_LENGTH) + { + my_errno=HA_ERR_WRONG_IN_RECORD; + DBUG_RETURN(1); + } + /* Check if next block is a delete block */ + del_block.second_read=0; + remove_next_block=0; + if (_mi_get_block_info(&del_block,info->dfile,filepos+length) & + BLOCK_DELETED && del_block.block_len+length < MI_DYN_MAX_BLOCK_LENGTH) + { + /* We can't remove this yet as this block may be the head block */ + remove_next_block=1; + length+=del_block.block_len; + } + + block_info.header[0]=0; + mi_int3store(block_info.header+1,length); + mi_sizestore(block_info.header+4,info->s->state.dellink); + if (b_type & BLOCK_LAST) + bfill(block_info.header+12,8,255); + else + mi_sizestore(block_info.header+12,block_info.next_filepos); + if (info->s->file_write(info,(uchar*) block_info.header,20,filepos, + MYF(MY_NABP))) + DBUG_RETURN(1); + info->s->state.dellink = filepos; + info->state->del++; + info->state->empty+=length; + filepos=block_info.next_filepos; + + /* Now it's safe to unlink the deleted block directly after this one */ + if (remove_next_block && unlink_deleted_block(info,&del_block)) + error=1; + } while (!(b_type & BLOCK_LAST)); + + DBUG_RETURN(error); +} + + + /* Write a block to datafile */ + +int _mi_write_part_record(MI_INFO *info, + my_off_t filepos, /* points at empty block */ + ulong length, /* length of block */ + my_off_t next_filepos,/* Next empty block */ + uchar **record, /* pointer to record ptr */ + ulong *reclength, /* length of *record */ + int *flag) /* *flag == 0 if header */ +{ + ulong head_length,res_length,extra_length,long_block,del_length; + uchar *pos,*record_end; + my_off_t next_delete_block; + uchar temp[MI_SPLIT_LENGTH+MI_DYN_DELETE_BLOCK_HEADER]; + DBUG_ENTER("_mi_write_part_record"); + + next_delete_block=HA_OFFSET_ERROR; + + res_length=extra_length=0; + if (length > *reclength + MI_SPLIT_LENGTH) + { /* Splitt big block */ + res_length=MY_ALIGN(length- *reclength - MI_EXTEND_BLOCK_LENGTH, + MI_DYN_ALIGN_SIZE); + length-= res_length; /* Use this for first part */ + } + long_block= (length < 65520L && *reclength < 65520L) ? 0 : 1; + if (length == *reclength+ 3 + long_block) + { + /* Block is exactly of the right length */ + temp[0]=(uchar) (1+ *flag)+(uchar) long_block; /* Flag is 0 or 6 */ + if (long_block) + { + mi_int3store(temp+1,*reclength); + head_length=4; + } + else + { + mi_int2store(temp+1,*reclength); + head_length=3; + } + } + else if (length-long_block < *reclength+4) + { /* To short block */ + if (next_filepos == HA_OFFSET_ERROR) + next_filepos= (info->s->state.dellink != HA_OFFSET_ERROR && + !info->append_insert_at_end ? + info->s->state.dellink : info->state->data_file_length); + if (*flag == 0) /* First block */ + { + if (*reclength > MI_MAX_BLOCK_LENGTH) + { + head_length= 16; + temp[0]=13; + mi_int4store(temp+1,*reclength); + mi_int3store(temp+5,length-head_length); + mi_sizestore((uchar*) temp+8,next_filepos); + } + else + { + head_length=5+8+long_block*2; + temp[0]=5+(uchar) long_block; + if (long_block) + { + mi_int3store(temp+1,*reclength); + mi_int3store(temp+4,length-head_length); + mi_sizestore((uchar*) temp+7,next_filepos); + } + else + { + mi_int2store(temp+1,*reclength); + mi_int2store(temp+3,length-head_length); + mi_sizestore((uchar*) temp+5,next_filepos); + } + } + } + else + { + head_length=3+8+long_block; + temp[0]=11+(uchar) long_block; + if (long_block) + { + mi_int3store(temp+1,length-head_length); + mi_sizestore((uchar*) temp+4,next_filepos); + } + else + { + mi_int2store(temp+1,length-head_length); + mi_sizestore((uchar*) temp+3,next_filepos); + } + } + } + else + { /* Block with empty info last */ + head_length=4+long_block; + extra_length= length- *reclength-head_length; + temp[0]= (uchar) (3+ *flag)+(uchar) long_block; /* 3,4 or 9,10 */ + if (long_block) + { + mi_int3store(temp+1,*reclength); + temp[4]= (uchar) (extra_length); + } + else + { + mi_int2store(temp+1,*reclength); + temp[3]= (uchar) (extra_length); + } + length= *reclength+head_length; /* Write only what is needed */ + } + DBUG_DUMP("header",(uchar*) temp,head_length); + + /* Make a long block for one write */ + record_end= *record+length-head_length; + del_length=(res_length ? MI_DYN_DELETE_BLOCK_HEADER : 0); + bmove((uchar*) (*record-head_length),(uchar*) temp,head_length); + memcpy(temp,record_end,(size_t) (extra_length+del_length)); + bzero((uchar*) record_end,extra_length); + + if (res_length) + { + /* Check first if we can join this block with the next one */ + MI_BLOCK_INFO del_block; + my_off_t next_block=filepos+length+extra_length+res_length; + + del_block.second_read=0; + if (next_block < info->state->data_file_length && + info->s->state.dellink != HA_OFFSET_ERROR) + { + if ((_mi_get_block_info(&del_block,info->dfile,next_block) + & BLOCK_DELETED) && + res_length + del_block.block_len < MI_DYN_MAX_BLOCK_LENGTH) + { + if (unlink_deleted_block(info,&del_block)) + goto err; + res_length+=del_block.block_len; + } + } + + /* Create a delete link of the last part of the block */ + pos=record_end+extra_length; + pos[0]= '\0'; + mi_int3store(pos+1,res_length); + mi_sizestore(pos+4,info->s->state.dellink); + bfill(pos+12,8,255); /* End link */ + next_delete_block=info->s->state.dellink; + info->s->state.dellink= filepos+length+extra_length; + info->state->del++; + info->state->empty+=res_length; + info->s->state.split++; + } + if (info->opt_flag & WRITE_CACHE_USED && + info->update & HA_STATE_WRITE_AT_END) + { + if (info->update & HA_STATE_EXTEND_BLOCK) + { + info->update&= ~HA_STATE_EXTEND_BLOCK; + if (my_block_write(&info->rec_cache,(uchar*) *record-head_length, + length+extra_length+del_length,filepos)) + goto err; + } + else if (my_b_write(&info->rec_cache,(uchar*) *record-head_length, + length+extra_length+del_length)) + goto err; + } + else + { + info->rec_cache.seek_not_done=1; + if (info->s->file_write(info,(uchar*) *record-head_length,length+extra_length+ + del_length,filepos,info->s->write_flag)) + goto err; + } + memcpy(record_end,temp,(size_t) (extra_length+del_length)); + *record=record_end; + *reclength-=(length-head_length); + *flag=6; + + if (del_length) + { + /* link the next delete block to this */ + if (update_backward_delete_link(info, next_delete_block, + info->s->state.dellink)) + goto err; + } + + DBUG_RETURN(0); +err: + DBUG_PRINT("exit",("errno: %d",my_errno)); + DBUG_RETURN(1); +} /*_mi_write_part_record */ + + + /* update record from datafile */ + +static int update_dynamic_record(MI_INFO *info, my_off_t filepos, uchar *record, + ulong reclength) +{ + int flag; + uint error; + ulong length; + MI_BLOCK_INFO block_info; + DBUG_ENTER("update_dynamic_record"); + + flag=block_info.second_read=0; + /* + Check if we have enough room for the record. + First we do simplified check to make usual case faster. + Then we do more precise check for the space left. + Though it still is not absolutely precise, as + we always use MI_MAX_DYN_BLOCK_HEADER while it can be + less in the most of the cases. + */ + + /* + compare with just the reclength as we're going + to get some space from the old replaced record + */ + if (unlikely(info->s->base.max_data_file_length - + info->state->data_file_length < reclength)) + { + /* + let's read the old record's block to find out the length of the + old record + */ + if ((error=_mi_get_block_info(&block_info,info->dfile,filepos)) + & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR | BLOCK_FATAL_ERROR)) + { + DBUG_PRINT("error",("Got wrong block info")); + if (!(error & BLOCK_FATAL_ERROR)) + my_errno=HA_ERR_WRONG_IN_RECORD; + goto err; + } + + /* + if new record isn't longer, we can go on safely + */ + if (block_info.rec_len < reclength) + { + if (info->s->base.max_data_file_length - info->state->data_file_length + + info->state->empty - info->state->del * MI_MAX_DYN_BLOCK_HEADER < + reclength - block_info.rec_len + MI_MAX_DYN_BLOCK_HEADER) + { + my_errno=HA_ERR_RECORD_FILE_FULL; + goto err; + } + } + block_info.second_read=0; + } + + while (reclength > 0) + { + if (filepos != info->s->state.dellink) + { + block_info.next_filepos= HA_OFFSET_ERROR; + if ((error=_mi_get_block_info(&block_info,info->dfile,filepos)) + & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR | + BLOCK_FATAL_ERROR)) + { + DBUG_PRINT("error",("Got wrong block info")); + if (!(error & BLOCK_FATAL_ERROR)) + my_errno=HA_ERR_WRONG_IN_RECORD; + goto err; + } + length=(ulong) (block_info.filepos-filepos) + block_info.block_len; + if (length < reclength) + { + uint tmp=MY_ALIGN(reclength - length + 3 + + MY_TEST(reclength >= 65520L), MI_DYN_ALIGN_SIZE); + /* Don't create a block bigger than MI_MAX_BLOCK_LENGTH */ + tmp= MY_MIN(length+tmp, MI_MAX_BLOCK_LENGTH)-length; + /* Check if we can extend this block */ + if (block_info.filepos + block_info.block_len == + info->state->data_file_length && + info->state->data_file_length < + info->s->base.max_data_file_length-tmp) + { + /* extend file */ + DBUG_PRINT("info",("Extending file with %d bytes",tmp)); + if (info->nextpos == info->state->data_file_length) + info->nextpos+= tmp; + info->state->data_file_length+= tmp; + info->update|= HA_STATE_WRITE_AT_END | HA_STATE_EXTEND_BLOCK; + length+=tmp; + } + else if (length < MI_MAX_BLOCK_LENGTH - MI_MIN_BLOCK_LENGTH) + { + /* + Check if next block is a deleted block + Above we have MI_MIN_BLOCK_LENGTH to avoid the problem where + the next block is so small it can't be split which could + cause problems + */ + + MI_BLOCK_INFO del_block; + del_block.second_read=0; + if (_mi_get_block_info(&del_block,info->dfile, + block_info.filepos + block_info.block_len) & + BLOCK_DELETED) + { + /* Use; Unlink it and extend the current block */ + DBUG_PRINT("info",("Extending current block")); + if (unlink_deleted_block(info,&del_block)) + goto err; + if ((length+=del_block.block_len) > MI_MAX_BLOCK_LENGTH) + { + /* + New block was too big, link overflow part back to + delete list + */ + my_off_t next_pos; + ulong rest_length= length-MI_MAX_BLOCK_LENGTH; + set_if_bigger(rest_length, MI_MIN_BLOCK_LENGTH); + next_pos= del_block.filepos+ del_block.block_len - rest_length; + + if (update_backward_delete_link(info, info->s->state.dellink, + next_pos)) + DBUG_RETURN(1); + + /* create delete link for data that didn't fit into the page */ + del_block.header[0]=0; + mi_int3store(del_block.header+1, rest_length); + mi_sizestore(del_block.header+4,info->s->state.dellink); + bfill(del_block.header+12,8,255); + if (info->s->file_write(info,(uchar*) del_block.header,20, next_pos, + MYF(MY_NABP))) + DBUG_RETURN(1); + info->s->state.dellink= next_pos; + info->s->state.split++; + info->state->del++; + info->state->empty+= rest_length; + length-= rest_length; + } + } + } + } + } + else + { + if (_mi_find_writepos(info,reclength,&filepos,&length)) + goto err; + } + if (_mi_write_part_record(info,filepos,length,block_info.next_filepos, + &record,&reclength,&flag)) + goto err; + if ((filepos=block_info.next_filepos) == HA_OFFSET_ERROR) + { + /* Start writing data on deleted blocks */ + filepos=info->s->state.dellink; + } + } + + if (block_info.next_filepos != HA_OFFSET_ERROR) + { + /* + delete_dynamic_record() may change data file position. + IO cache must be notified as it may still have cached + data, which has to be flushed later. + */ + info->rec_cache.seek_not_done= 1; + if (delete_dynamic_record(info,block_info.next_filepos,1)) + goto err; + } + DBUG_RETURN(0); +err: + DBUG_RETURN(1); +} + + + /* Pack a record. Return new reclength */ + +uint _mi_rec_pack(MI_INFO *info, register uchar *to, + register const uchar *from) +{ + uint length,new_length,flag,bit,i; + uchar *pos,*end,*startpos,*packpos; + enum en_fieldtype type; + reg3 MI_COLUMNDEF *rec; + MI_BLOB *blob; + DBUG_ENTER("_mi_rec_pack"); + + flag=0 ; bit=1; + startpos=packpos=to; to+= info->s->base.pack_bits; blob=info->blobs; + rec=info->s->rec; + + for (i=info->s->base.fields ; i-- > 0; from+= length,rec++) + { + length=(uint) rec->length; + if ((type = (enum en_fieldtype) rec->type) != FIELD_NORMAL) + { + if (type == FIELD_BLOB) + { + if (!blob->length) + flag|=bit; + else + { + char *temp_pos; + size_t tmp_length=length-portable_sizeof_char_ptr; + memcpy((uchar*) to,from,tmp_length); + memcpy(&temp_pos,from+tmp_length,sizeof(char*)); + memcpy(to+tmp_length,temp_pos,(size_t) blob->length); + to+=tmp_length+blob->length; + } + blob++; + } + else if (type == FIELD_SKIP_ZERO) + { + if (memcmp((uchar*) from,zero_string,length) == 0) + flag|=bit; + else + { + memcpy((uchar*) to,from,(size_t) length); to+=length; + } + } + else if (type == FIELD_SKIP_ENDSPACE || + type == FIELD_SKIP_PRESPACE) + { + pos= (uchar*) from; end= (uchar*) from + length; + if (type == FIELD_SKIP_ENDSPACE) + { /* Pack trailing spaces */ + while (end > from && *(end-1) == ' ') + end--; + } + else + { /* Pack pref-spaces */ + while (pos < end && *pos == ' ') + pos++; + } + new_length=(uint) (end-pos); + if (new_length + 1 + MY_TEST(rec->length > 255 && new_length > 127) + < length) + { + if (rec->length > 255 && new_length > 127) + { + to[0]= (uchar) ((new_length & 127) + 128); + to[1]= (uchar) (new_length >> 7); + to+=2; + } + else + *to++= (uchar) new_length; + memcpy((uchar*) to,pos,(size_t) new_length); to+=new_length; + flag|=bit; + } + else + { + memcpy(to,from,(size_t) length); to+=length; + } + } + else if (type == FIELD_VARCHAR) + { + uint pack_length= HA_VARCHAR_PACKLENGTH(rec->length -1); + uint tmp_length; + if (pack_length == 1) + { + tmp_length= (uint) *(uchar*) from; + *to++= *from; + } + else + { + tmp_length= uint2korr(from); + store_key_length_inc(to,tmp_length); + } + memcpy(to, from+pack_length,tmp_length); + to+= tmp_length; + continue; + } + else + { + memcpy(to,from,(size_t) length); to+=length; + continue; /* Normal field */ + } + if ((bit= bit << 1) >= 256) + { + *packpos++= (uchar) flag; + bit=1; flag=0; + } + } + else + { + memcpy(to,from,(size_t) length); to+=length; + } + } + if (bit != 1) + *packpos= (uchar) flag; + if (info->s->calc_checksum) + *to++= (uchar) info->checksum; + DBUG_PRINT("exit",("packed length: %d",(int) (to-startpos))); + DBUG_RETURN((uint) (to-startpos)); +} /* _mi_rec_pack */ + + + +/* + Check if a record was correctly packed. Used only by myisamchk + Returns 0 if record is ok. +*/ + +my_bool _mi_rec_check(MI_INFO *info,const uchar *record, uchar *rec_buff, + ulong packed_length, my_bool with_checksum) +{ + uint length,new_length,flag,bit,i; + uchar *pos,*end,*packpos,*to; + enum en_fieldtype type; + reg3 MI_COLUMNDEF *rec; + DBUG_ENTER("_mi_rec_check"); + + packpos=rec_buff; to= rec_buff+info->s->base.pack_bits; + rec=info->s->rec; + flag= *packpos; bit=1; + + for (i=info->s->base.fields ; i-- > 0; record+= length, rec++) + { + length=(uint) rec->length; + if ((type = (enum en_fieldtype) rec->type) != FIELD_NORMAL) + { + if (type == FIELD_BLOB) + { + uint blob_length= + _mi_calc_blob_length(length-portable_sizeof_char_ptr,record); + if (!blob_length && !(flag & bit)) + goto err; + if (blob_length) + to+=length - portable_sizeof_char_ptr+ blob_length; + } + else if (type == FIELD_SKIP_ZERO) + { + if (memcmp((uchar*) record,zero_string,length) == 0) + { + if (!(flag & bit)) + goto err; + } + else + to+=length; + } + else if (type == FIELD_SKIP_ENDSPACE || + type == FIELD_SKIP_PRESPACE) + { + pos= (uchar*) record; end= (uchar*) record + length; + if (type == FIELD_SKIP_ENDSPACE) + { /* Pack trailing spaces */ + while (end > record && *(end-1) == ' ') + end--; + } + else + { /* Pack pre-spaces */ + while (pos < end && *pos == ' ') + pos++; + } + new_length=(uint) (end-pos); + if (new_length + 1 + MY_TEST(rec->length > 255 && new_length > 127) + < length) + { + if (!(flag & bit)) + goto err; + if (rec->length > 255 && new_length > 127) + { + /* purecov: begin inspected */ + if (to[0] != (uchar) ((new_length & 127) + 128) || + to[1] != (uchar) (new_length >> 7)) + goto err; + to+=2; + /* purecov: end */ + } + else if (*to++ != (uchar) new_length) + goto err; + to+=new_length; + } + else + to+=length; + } + else if (type == FIELD_VARCHAR) + { + uint pack_length= HA_VARCHAR_PACKLENGTH(rec->length -1); + uint tmp_length; + if (pack_length == 1) + { + tmp_length= (uint) *(uchar*) record; + to+= 1+ tmp_length; + continue; + } + else + { + tmp_length= uint2korr(record); + to+= get_pack_length(tmp_length)+tmp_length; + } + continue; + } + else + { + to+=length; + continue; /* Normal field */ + } + if ((bit= bit << 1) >= 256) + { + flag= *++packpos; + bit=1; + } + } + else + to+= length; + } + if (packed_length != (uint) (to - rec_buff) + MY_TEST(info->s->calc_checksum) || + (bit != 1 && (flag & ~(bit - 1)))) + goto err; + if (with_checksum && ((uchar) info->checksum != (uchar) *to)) + { + DBUG_PRINT("error",("wrong checksum for row")); + goto err; + } + DBUG_RETURN(0); + +err: + DBUG_RETURN(1); +} + + + + /* Unpacks a record */ + /* Returns -1 and my_errno =HA_ERR_RECORD_DELETED if reclength isn't */ + /* right. Returns reclength (>0) if ok */ + +size_t _mi_rec_unpack(register MI_INFO *info, register uchar *to, uchar *from, + ulong found_length) +{ + uint flag,bit,length,rec_length,min_pack_length; + enum en_fieldtype type; + uchar *from_end,*to_end,*packpos; + reg3 MI_COLUMNDEF *rec,*end_field; + DBUG_ENTER("_mi_rec_unpack"); + + to_end=to + info->s->base.reclength; + from_end=from+found_length; + flag= (uchar) *from; bit=1; packpos=from; + if (found_length < info->s->base.min_pack_length) + goto err; + from+= info->s->base.pack_bits; + min_pack_length=info->s->base.min_pack_length - info->s->base.pack_bits; + + for (rec=info->s->rec , end_field=rec+info->s->base.fields ; + rec < end_field ; to+= rec_length, rec++) + { + rec_length=rec->length; + if ((type = (enum en_fieldtype) rec->type) != FIELD_NORMAL && + (type != FIELD_CHECK)) + { + if (type == FIELD_VARCHAR) + { + uint pack_length= HA_VARCHAR_PACKLENGTH(rec_length-1); + if (pack_length == 1) + { + length= (uint) *(uchar*) from; + if (length > rec_length-1) + goto err; + *to= *from++; + } + else + { + get_key_length(length, from); + if (length > rec_length-2) + goto err; + int2store(to,length); + } + if (from+length > from_end) + goto err; + memcpy(to+pack_length, from, length); + from+= length; + min_pack_length--; + continue; + } + if (flag & bit) + { + if (type == FIELD_BLOB || type == FIELD_SKIP_ZERO) + bzero((uchar*) to,rec_length); + else if (type == FIELD_SKIP_ENDSPACE || + type == FIELD_SKIP_PRESPACE) + { + if (rec->length > 255 && *from & 128) + { + if (from + 1 >= from_end) + goto err; + length= (*from & 127)+ ((uint) (uchar) *(from+1) << 7); from+=2; + } + else + { + if (from == from_end) + goto err; + length= (uchar) *from++; + } + min_pack_length--; + if (length >= rec_length || + min_pack_length + length > (uint) (from_end - from)) + goto err; + if (type == FIELD_SKIP_ENDSPACE) + { + memcpy(to,(uchar*) from,(size_t) length); + bfill((uchar*) to+length,rec_length-length,' '); + } + else + { + bfill((uchar*) to,rec_length-length,' '); + memcpy(to+rec_length-length,(uchar*) from,(size_t) length); + } + from+=length; + } + } + else if (type == FIELD_BLOB) + { + uint size_length=rec_length- portable_sizeof_char_ptr; + ulong blob_length=_mi_calc_blob_length(size_length,from); + ulong from_left= (ulong) (from_end - from); + if (from_left < size_length || + from_left - size_length < blob_length || + from_left - size_length - blob_length < min_pack_length) + goto err; + memcpy(to, from, (size_t) size_length); + from+=size_length; + memcpy(to+size_length, &from, sizeof(char*)); + from+=blob_length; + } + else + { + if (type == FIELD_SKIP_ENDSPACE || type == FIELD_SKIP_PRESPACE) + min_pack_length--; + if (min_pack_length + rec_length > (uint) (from_end - from)) + goto err; + memcpy(to,(uchar*) from,(size_t) rec_length); from+=rec_length; + } + if ((bit= bit << 1) >= 256) + { + flag= (uchar) *++packpos; bit=1; + } + } + else + { + if (min_pack_length > (uint) (from_end - from)) + goto err; + min_pack_length-=rec_length; + memcpy(to, (uchar*) from, (size_t) rec_length); + from+=rec_length; + } + } + if (info->s->calc_checksum) + from++; + if (to == to_end && from == from_end && (bit == 1 || !(flag & ~(bit-1)))) + DBUG_RETURN(found_length); + +err: + my_errno= HA_ERR_WRONG_IN_RECORD; + DBUG_PRINT("error",("to_end: %p -> %p from_end: %p -> %p", + to, to_end, from, from_end)); + DBUG_DUMP("from",(uchar*) info->rec_buff,info->s->base.min_pack_length); + DBUG_RETURN(MY_FILE_ERROR); +} /* _mi_rec_unpack */ + + + /* Calc length of blob. Update info in blobs->length */ + +ulong _mi_calc_total_blob_length(MI_INFO *info, const uchar *record) +{ + ulong length; + MI_BLOB *blob,*end; + + for (length=0, blob= info->blobs, end=blob+info->s->base.blobs ; + blob != end; + blob++) + { + blob->length=_mi_calc_blob_length(blob->pack_length,record + blob->offset); + length+=blob->length; + } + return length; +} + + +ulong _mi_calc_blob_length(uint length, const uchar *pos) +{ + switch (length) { + case 1: + return (uint) (uchar) *pos; + case 2: + return (uint) uint2korr(pos); + case 3: + return uint3korr(pos); + case 4: + return uint4korr(pos); + default: + break; + } + return 0; /* Impossible */ +} + + +void _mi_store_blob_length(uchar *pos,uint pack_length,uint length) +{ + switch (pack_length) { + case 1: + *pos= (uchar) length; + break; + case 2: + int2store(pos,length); + break; + case 3: + int3store(pos,length); + break; + case 4: + int4store(pos,length); + default: + break; + } + return; +} + + +/* + Read record from datafile. + + SYNOPSIS + _mi_read_dynamic_record() + info MI_INFO pointer to table. + filepos From where to read the record. + buf Destination for record. + + NOTE + + If a write buffer is active, it needs to be flushed if its contents + intersects with the record to read. We always check if the position + of the first byte of the write buffer is lower than the position + past the last byte to read. In theory this is also true if the write + buffer is completely below the read segment. That is, if there is no + intersection. But this case is unusual. We flush anyway. Only if the + first byte in the write buffer is above the last byte to read, we do + not flush. + + A dynamic record may need several reads. So this check must be done + before every read. Reading a dynamic record starts with reading the + block header. If the record does not fit into the free space of the + header, the block may be longer than the header. In this case a + second read is necessary. These one or two reads repeat for every + part of the record. + + RETURN + 0 OK + -1 Error +*/ + +int _mi_read_dynamic_record(MI_INFO *info, my_off_t filepos, uchar *buf) +{ + int block_of_record; + uint b_type,UNINIT_VAR(left_length); + uchar *UNINIT_VAR(to); + MI_BLOCK_INFO block_info; + File file; + DBUG_ENTER("mi_read_dynamic_record"); + + if (filepos != HA_OFFSET_ERROR) + { + file=info->dfile; + block_of_record= 0; /* First block of record is numbered as zero. */ + block_info.second_read= 0; + do + { + /* A corrupted table can have wrong pointers. (Bug# 19835) */ + if (filepos == HA_OFFSET_ERROR) + goto panic; + if (info->opt_flag & WRITE_CACHE_USED && + info->rec_cache.pos_in_file < filepos + MI_BLOCK_INFO_HEADER_LENGTH && + flush_io_cache(&info->rec_cache)) + goto err; + info->rec_cache.seek_not_done=1; + if ((b_type= _mi_get_block_info(&block_info, file, filepos)) + & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR | + BLOCK_FATAL_ERROR)) + { + if (b_type & (BLOCK_SYNC_ERROR | BLOCK_DELETED)) + my_errno=HA_ERR_RECORD_DELETED; + goto err; + } + if (block_of_record++ == 0) /* First block */ + { + if (block_info.rec_len > (uint) info->s->base.max_pack_length) + goto panic; + if (info->s->base.blobs) + { + if (!(to=mi_alloc_rec_buff(info, block_info.rec_len, + &info->rec_buff))) + goto err; + } + else + to= info->rec_buff; + left_length=block_info.rec_len; + } + if (left_length < block_info.data_len || ! block_info.data_len) + goto panic; /* Wrong linked record */ + /* copy information that is already read */ + { + uint offset= (uint) (block_info.filepos - filepos); + uint prefetch_len= (sizeof(block_info.header) - offset); + filepos+= sizeof(block_info.header); + + if (prefetch_len > block_info.data_len) + prefetch_len= block_info.data_len; + if (prefetch_len) + { + memcpy((uchar*) to, block_info.header + offset, prefetch_len); + block_info.data_len-= prefetch_len; + left_length-= prefetch_len; + to+= prefetch_len; + } + } + /* read rest of record from file */ + if (block_info.data_len) + { + if (info->opt_flag & WRITE_CACHE_USED && + info->rec_cache.pos_in_file < filepos + block_info.data_len && + flush_io_cache(&info->rec_cache)) + goto err; + /* + What a pity that this method is not called 'file_pread' and that + there is no equivalent without seeking. We are at the right + position already. :( + */ + if (info->s->file_read(info, (uchar*) to, block_info.data_len, + filepos, MYF(MY_NABP))) + goto panic; + left_length-=block_info.data_len; + to+=block_info.data_len; + } + filepos= block_info.next_filepos; + } while (left_length); + + info->update|= HA_STATE_AKTIV; /* We have a aktive record */ + fast_mi_writeinfo(info); + DBUG_RETURN(_mi_rec_unpack(info,buf,info->rec_buff,block_info.rec_len) != + MY_FILE_ERROR ? 0 : -1); + } + fast_mi_writeinfo(info); + DBUG_RETURN(-1); /* Wrong data to read */ + +panic: + my_errno=HA_ERR_WRONG_IN_RECORD; +err: + (void) _mi_writeinfo(info,0); + DBUG_RETURN(-1); +} + + /* compare unique constraint between stored rows */ + +int _mi_cmp_dynamic_unique(MI_INFO *info, MI_UNIQUEDEF *def, + const uchar *record, my_off_t pos) +{ + uchar *rec_buff,*old_record; + int error; + DBUG_ENTER("_mi_cmp_dynamic_unique"); + + if (!(old_record=my_alloca(info->s->base.reclength))) + DBUG_RETURN(1); + + /* Don't let the compare destroy blobs that may be in use */ + rec_buff=info->rec_buff; + if (info->s->base.blobs) + info->rec_buff=0; + error=_mi_read_dynamic_record(info,pos,old_record); + if (!error) + error=mi_unique_comp(def, record, old_record, def->null_are_equal); + if (info->s->base.blobs) + { + my_free(mi_get_rec_buff_ptr(info, info->rec_buff)); + info->rec_buff=rec_buff; + } + my_afree(old_record); + DBUG_RETURN(error); +} + + + /* Compare of record one disk with packed record in memory */ + +int _mi_cmp_dynamic_record(register MI_INFO *info, register const uchar *record) +{ + uint flag,reclength,b_type; + my_off_t filepos; + uchar *buffer; + MI_BLOCK_INFO block_info; + DBUG_ENTER("_mi_cmp_dynamic_record"); + + if (info->opt_flag & WRITE_CACHE_USED) + { + info->update&= ~(HA_STATE_WRITE_AT_END | HA_STATE_EXTEND_BLOCK); + if (flush_io_cache(&info->rec_cache)) + DBUG_RETURN(-1); + } + info->rec_cache.seek_not_done=1; + + /* If nobody have touched the database we don't have to test rec */ + + buffer=info->rec_buff; + if ((info->opt_flag & READ_CHECK_USED)) + { /* If check isn't disabled */ + if (info->s->base.blobs) + { + if (!(buffer=(uchar*) my_alloca(info->s->base.pack_reclength+ + _mi_calc_total_blob_length(info,record)))) + DBUG_RETURN(-1); + } + reclength=_mi_rec_pack(info,buffer,record); + record= buffer; + + filepos=info->lastpos; + flag=block_info.second_read=0; + block_info.next_filepos=filepos; + while (reclength > 0) + { + if ((b_type=_mi_get_block_info(&block_info,info->dfile, + block_info.next_filepos)) + & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR | + BLOCK_FATAL_ERROR)) + { + if (b_type & (BLOCK_SYNC_ERROR | BLOCK_DELETED)) + my_errno=HA_ERR_RECORD_CHANGED; + goto err; + } + if (flag == 0) /* First block */ + { + flag=1; + if (reclength != block_info.rec_len) + { + my_errno=HA_ERR_RECORD_CHANGED; + goto err; + } + } else if (reclength < block_info.data_len) + { + my_errno=HA_ERR_WRONG_IN_RECORD; + goto err; + } + reclength-=block_info.data_len; + if (_mi_cmp_buffer(info->dfile,record,block_info.filepos, + block_info.data_len)) + { + my_errno=HA_ERR_RECORD_CHANGED; + goto err; + } + flag=1; + record+=block_info.data_len; + } + } + my_errno=0; +err: + if (buffer != info->rec_buff) + my_afree((uchar*) buffer); + DBUG_RETURN(my_errno); +} + + + /* Compare file to buffert */ + +static int _mi_cmp_buffer(File file, const uchar *buff, my_off_t filepos, + uint length) +{ + uint next_length; + uchar temp_buff[IO_SIZE*2]; + DBUG_ENTER("_mi_cmp_buffer"); + + next_length= IO_SIZE*2 - (uint) (filepos & (IO_SIZE-1)); + + while (length > IO_SIZE*2) + { + if (mysql_file_pread(file, temp_buff, next_length, filepos, MYF(MY_NABP)) || + memcmp(buff, temp_buff, next_length)) + goto err; + filepos+=next_length; + buff+=next_length; + length-= next_length; + next_length=IO_SIZE*2; + } + if (mysql_file_pread(file, temp_buff, length, filepos, MYF(MY_NABP))) + goto err; + DBUG_RETURN(memcmp(buff,temp_buff,length)); +err: + DBUG_RETURN(1); +} + + +/* + Read record from datafile. + + SYNOPSIS + _mi_read_rnd_dynamic_record() + info MI_INFO pointer to table. + buf Destination for record. + filepos From where to read the record. + skip_deleted_blocks If to repeat reading until a non-deleted + record is found. + + NOTE + + If a write buffer is active, it needs to be flushed if its contents + intersects with the record to read. We always check if the position + of the first byte of the write buffer is lower than the position + past the last byte to read. In theory this is also true if the write + buffer is completely below the read segment. That is, if there is no + intersection. But this case is unusual. We flush anyway. Only if the + first byte in the write buffer is above the last byte to read, we do + not flush. + + A dynamic record may need several reads. So this check must be done + before every read. Reading a dynamic record starts with reading the + block header. If the record does not fit into the free space of the + header, the block may be longer than the header. In this case a + second read is necessary. These one or two reads repeat for every + part of the record. + + RETURN + 0 OK + != 0 Error +*/ + +int _mi_read_rnd_dynamic_record(MI_INFO *info, uchar *buf, + register my_off_t filepos, + my_bool skip_deleted_blocks) +{ + int block_of_record, info_read, save_errno; + uint left_len,b_type; + uchar *UNINIT_VAR(to); + MI_BLOCK_INFO block_info; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("_mi_read_rnd_dynamic_record"); + + info_read=0; + + if (info->lock_type == F_UNLCK) + { +#ifndef UNSAFE_LOCKING + if (share->tot_locks == 0) + { + if (my_lock(share->kfile,F_RDLCK,0L,F_TO_EOF, + MYF(MY_SEEK_NOT_DONE) | info->lock_wait)) + DBUG_RETURN(my_errno); + } +#else + info->tmp_lock_type=F_RDLCK; +#endif + } + else + info_read=1; /* memory-keyinfoblock is ok */ + + block_of_record= 0; /* First block of record is numbered as zero. */ + block_info.second_read= 0; + left_len=1; + do + { + if (filepos >= info->state->data_file_length) + { + if (!info_read) + { /* Check if changed */ + info_read=1; + info->rec_cache.seek_not_done=1; + if (mi_state_info_read_dsk(share->kfile,&share->state,1)) + goto panic; + } + if (filepos >= info->state->data_file_length) + { + my_errno= HA_ERR_END_OF_FILE; + goto err; + } + } + if (info->opt_flag & READ_CACHE_USED) + { + if (_mi_read_cache(&info->rec_cache,(uchar*) block_info.header,filepos, + sizeof(block_info.header), + (!block_of_record && skip_deleted_blocks ? + READING_NEXT : 0) | READING_HEADER)) + goto panic; + b_type=_mi_get_block_info(&block_info,-1,filepos); + } + else + { + if (info->opt_flag & WRITE_CACHE_USED && + info->rec_cache.pos_in_file < filepos + MI_BLOCK_INFO_HEADER_LENGTH && + flush_io_cache(&info->rec_cache)) + DBUG_RETURN(my_errno); + info->rec_cache.seek_not_done=1; + b_type=_mi_get_block_info(&block_info,info->dfile,filepos); + } + + if (b_type & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR | + BLOCK_FATAL_ERROR)) + { + if ((b_type & (BLOCK_DELETED | BLOCK_SYNC_ERROR))) + { + if (skip_deleted_blocks) + { + filepos=block_info.filepos+block_info.block_len; + block_info.second_read=0; + continue; /* Search after next_record */ + } + /* + If we're not on the first block of a record and + the block is marked as deleted or out of sync, + something's gone wrong: the record is damaged. + */ + if (block_of_record != 0) + goto panic; + my_errno=HA_ERR_RECORD_DELETED; + info->lastpos=block_info.filepos; + info->nextpos=block_info.filepos+block_info.block_len; + } + goto err; + } + if (block_of_record == 0) /* First block */ + { + if (block_info.rec_len > (uint) share->base.max_pack_length) + goto panic; + info->lastpos=filepos; + if (share->base.blobs) + { + if (!(to= mi_alloc_rec_buff(info, block_info.rec_len, + &info->rec_buff))) + goto err; + } + else + to= info->rec_buff; + left_len=block_info.rec_len; + } + if (left_len < block_info.data_len) + goto panic; /* Wrong linked record */ + + /* copy information that is already read */ + { + uint offset=(uint) (block_info.filepos - filepos); + uint tmp_length= (sizeof(block_info.header) - offset); + filepos=block_info.filepos; + + if (tmp_length > block_info.data_len) + tmp_length= block_info.data_len; + if (tmp_length) + { + memcpy((uchar*) to, block_info.header+offset,tmp_length); + block_info.data_len-=tmp_length; + left_len-=tmp_length; + to+=tmp_length; + filepos+=tmp_length; + } + } + /* read rest of record from file */ + if (block_info.data_len) + { + if (info->opt_flag & READ_CACHE_USED) + { + if (_mi_read_cache(&info->rec_cache,(uchar*) to,filepos, + block_info.data_len, + (!block_of_record && skip_deleted_blocks) ? + READING_NEXT : 0)) + goto panic; + } + else + { + if (info->opt_flag & WRITE_CACHE_USED && + info->rec_cache.pos_in_file < + block_info.filepos + block_info.data_len && + flush_io_cache(&info->rec_cache)) + goto err; + /* mysql_file_seek(info->dfile, filepos, MY_SEEK_SET, MYF(0)); */ + if (mysql_file_read(info->dfile, (uchar*) to, block_info.data_len, + MYF(MY_NABP))) + { + if (my_errno == HA_ERR_FILE_TOO_SHORT) + my_errno= HA_ERR_WRONG_IN_RECORD; /* Unexpected end of file */ + goto err; + } + } + } + /* + Increment block-of-record counter. If it was the first block, + remember the position behind the block for the next call. + */ + if (block_of_record++ == 0) + { + info->nextpos= block_info.filepos + block_info.block_len; + skip_deleted_blocks= 0; + } + left_len-=block_info.data_len; + to+=block_info.data_len; + filepos=block_info.next_filepos; + } while (left_len); + + info->update|= HA_STATE_AKTIV | HA_STATE_KEY_CHANGED; + fast_mi_writeinfo(info); + if (_mi_rec_unpack(info,buf,info->rec_buff,block_info.rec_len) != + MY_FILE_ERROR) + DBUG_RETURN(0); + DBUG_RETURN(my_errno); /* Wrong record */ + +panic: + my_errno=HA_ERR_WRONG_IN_RECORD; /* Something is fatal wrong */ +err: + save_errno=my_errno; + (void) _mi_writeinfo(info,0); + DBUG_RETURN(my_errno=save_errno); +} + + + /* Read and process header from a dynamic-record-file */ + +uint _mi_get_block_info(MI_BLOCK_INFO *info, File file, my_off_t filepos) +{ + uint return_val=0; + uchar *header=info->header; + + if (file >= 0) + { + /* + We do not use mysql_file_pread() here because we want to have the file + pointer set to the end of the header after this function. + mysql_file_pread() may leave the file pointer untouched. + */ + mysql_file_seek(file, filepos, MY_SEEK_SET, MYF(0)); + if (mysql_file_read(file, header, sizeof(info->header), MYF(0)) != + sizeof(info->header)) + goto err; + } + DBUG_DUMP("header",header,MI_BLOCK_INFO_HEADER_LENGTH); + if (info->second_read) + { + if (info->header[0] <= 6 || info->header[0] == 13) + return_val=BLOCK_SYNC_ERROR; + } + else + { + if (info->header[0] > 6 && info->header[0] != 13) + return_val=BLOCK_SYNC_ERROR; + } + info->next_filepos= HA_OFFSET_ERROR; /* Dummy if no next block */ + + switch (info->header[0]) { + case 0: + if ((info->block_len=(uint) mi_uint3korr(header+1)) < + MI_MIN_BLOCK_LENGTH || + (info->block_len & (MI_DYN_ALIGN_SIZE -1))) + goto err; + info->filepos=filepos; + info->next_filepos=mi_sizekorr(header+4); + info->prev_filepos=mi_sizekorr(header+12); +#if SIZEOF_OFF_T == 4 + if ((mi_uint4korr(header+4) != 0 && + (mi_uint4korr(header+4) != (ulong) ~0 || + info->next_filepos != (ulong) ~0)) || + (mi_uint4korr(header+12) != 0 && + (mi_uint4korr(header+12) != (ulong) ~0 || + info->prev_filepos != (ulong) ~0))) + goto err; +#endif + return return_val | BLOCK_DELETED; /* Deleted block */ + + case 1: + info->rec_len=info->data_len=info->block_len=mi_uint2korr(header+1); + info->filepos=filepos+3; + return return_val | BLOCK_FIRST | BLOCK_LAST; + case 2: + info->rec_len=info->data_len=info->block_len=mi_uint3korr(header+1); + info->filepos=filepos+4; + return return_val | BLOCK_FIRST | BLOCK_LAST; + + case 13: + info->rec_len=mi_uint4korr(header+1); + info->block_len=info->data_len=mi_uint3korr(header+5); + info->next_filepos=mi_sizekorr(header+8); + info->second_read=1; + info->filepos=filepos+16; + return return_val | BLOCK_FIRST; + + case 3: + info->rec_len=info->data_len=mi_uint2korr(header+1); + info->block_len=info->rec_len+ (uint) header[3]; + info->filepos=filepos+4; + return return_val | BLOCK_FIRST | BLOCK_LAST; + case 4: + info->rec_len=info->data_len=mi_uint3korr(header+1); + info->block_len=info->rec_len+ (uint) header[4]; + info->filepos=filepos+5; + return return_val | BLOCK_FIRST | BLOCK_LAST; + + case 5: + info->rec_len=mi_uint2korr(header+1); + info->block_len=info->data_len=mi_uint2korr(header+3); + info->next_filepos=mi_sizekorr(header+5); + info->second_read=1; + info->filepos=filepos+13; + return return_val | BLOCK_FIRST; + case 6: + info->rec_len=mi_uint3korr(header+1); + info->block_len=info->data_len=mi_uint3korr(header+4); + info->next_filepos=mi_sizekorr(header+7); + info->second_read=1; + info->filepos=filepos+15; + return return_val | BLOCK_FIRST; + + /* The following blocks are identical to 1-6 without rec_len */ + case 7: + info->data_len=info->block_len=mi_uint2korr(header+1); + info->filepos=filepos+3; + return return_val | BLOCK_LAST; + case 8: + info->data_len=info->block_len=mi_uint3korr(header+1); + info->filepos=filepos+4; + return return_val | BLOCK_LAST; + + case 9: + info->data_len=mi_uint2korr(header+1); + info->block_len=info->data_len+ (uint) header[3]; + info->filepos=filepos+4; + return return_val | BLOCK_LAST; + case 10: + info->data_len=mi_uint3korr(header+1); + info->block_len=info->data_len+ (uint) header[4]; + info->filepos=filepos+5; + return return_val | BLOCK_LAST; + + case 11: + info->data_len=info->block_len=mi_uint2korr(header+1); + info->next_filepos=mi_sizekorr(header+3); + info->second_read=1; + info->filepos=filepos+11; + return return_val; + case 12: + info->data_len=info->block_len=mi_uint3korr(header+1); + info->next_filepos=mi_sizekorr(header+4); + info->second_read=1; + info->filepos=filepos+12; + return return_val; + } + +err: + my_errno=HA_ERR_WRONG_IN_RECORD; /* Garbage */ + return BLOCK_ERROR; +} diff --git a/storage/myisam/mi_extra.c b/storage/myisam/mi_extra.c new file mode 100644 index 00000000..66238745 --- /dev/null +++ b/storage/myisam/mi_extra.c @@ -0,0 +1,451 @@ +/* + Copyright (c) 2000, 2010, Oracle and/or its affiliates + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "myisamdef.h" + +static void mi_extra_keyflag(MI_INFO *info, enum ha_extra_function function); + + +/* + Set options and buffers to optimize table handling + + SYNOPSIS + mi_extra() + info open table + function operation + extra_arg Pointer to extra argument (normally pointer to ulong) + Used when function is one of: + HA_EXTRA_WRITE_CACHE + HA_EXTRA_CACHE + RETURN VALUES + 0 ok + # error +*/ + +int mi_extra(MI_INFO *info, enum ha_extra_function function, void *extra_arg) +{ + int error=0; + ulong cache_size; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("mi_extra"); + DBUG_PRINT("enter",("function: %d",(int) function)); + + switch (function) { + case HA_EXTRA_RESET_STATE: /* Reset state (don't free buffers) */ + info->lastinx= 0; /* Use first index as def */ + info->last_search_keypage=info->lastpos= HA_OFFSET_ERROR; + info->page_changed=1; + /* Next/prev gives first/last */ + if (info->opt_flag & READ_CACHE_USED) + { + reinit_io_cache(&info->rec_cache,READ_CACHE,0, + (pbool) (info->lock_type != F_UNLCK), + (pbool) MY_TEST(info->update & HA_STATE_ROW_CHANGED) + ); + } + info->update= ((info->update & HA_STATE_CHANGED) | HA_STATE_NEXT_FOUND | + HA_STATE_PREV_FOUND); + break; + case HA_EXTRA_CACHE: + if (info->lock_type == F_UNLCK && + (share->options & HA_OPTION_PACK_RECORD)) + { + error=1; /* Not possibly if not locked */ + my_errno=EACCES; + break; + } + if (info->s->file_map) /* Don't use cache if mmap */ + break; +#if defined(HAVE_MMAP) && defined(HAVE_MADVISE) + if ((share->options & HA_OPTION_COMPRESS_RECORD)) + { + mysql_mutex_lock(&share->intern_lock); + if (_mi_memmap_file(info)) + { + /* We don't nead MADV_SEQUENTIAL if small file */ + madvise((char*) share->file_map, share->state.state.data_file_length, + share->state.state.data_file_length <= RECORD_CACHE_SIZE*16 ? + MADV_RANDOM : MADV_SEQUENTIAL); + mysql_mutex_unlock(&share->intern_lock); + break; + } + mysql_mutex_unlock(&share->intern_lock); + } +#endif + if (info->opt_flag & WRITE_CACHE_USED) + { + info->opt_flag&= ~WRITE_CACHE_USED; + if ((error=end_io_cache(&info->rec_cache))) + break; + } + if (!(info->opt_flag & + (READ_CACHE_USED | WRITE_CACHE_USED | MEMMAP_USED))) + { + cache_size= (extra_arg ? *(ulong*) extra_arg : + my_default_record_cache_size); + if (!(init_io_cache(&info->rec_cache,info->dfile, + (uint) MY_MIN(info->state->data_file_length+1, + cache_size), + READ_CACHE,0L,(pbool) (info->lock_type != F_UNLCK), + MYF(share->write_flag & MY_WAIT_IF_FULL)))) + { + info->opt_flag|=READ_CACHE_USED; + info->update&= ~HA_STATE_ROW_CHANGED; + } + if (share->concurrent_insert) + info->rec_cache.end_of_file=info->state->data_file_length; + } + break; + case HA_EXTRA_REINIT_CACHE: + if (info->opt_flag & READ_CACHE_USED) + { + reinit_io_cache(&info->rec_cache,READ_CACHE,info->nextpos, + (pbool) (info->lock_type != F_UNLCK), + (pbool) MY_TEST(info->update & HA_STATE_ROW_CHANGED)); + info->update&= ~HA_STATE_ROW_CHANGED; + if (share->concurrent_insert) + info->rec_cache.end_of_file=info->state->data_file_length; + } + break; + case HA_EXTRA_WRITE_CACHE: + if (info->lock_type == F_UNLCK) + { + error=1; /* Not possibly if not locked */ + break; + } + + cache_size= (extra_arg ? *(ulong*) extra_arg : + my_default_record_cache_size); + if (!(info->opt_flag & + (READ_CACHE_USED | WRITE_CACHE_USED | OPT_NO_ROWS)) && + !share->state.header.uniques) + if (!(init_io_cache(&info->rec_cache,info->dfile, cache_size, + WRITE_CACHE,info->state->data_file_length, + (pbool) (info->lock_type != F_UNLCK), + MYF(share->write_flag & MY_WAIT_IF_FULL)))) + { + info->opt_flag|=WRITE_CACHE_USED; + info->update&= ~(HA_STATE_ROW_CHANGED | + HA_STATE_WRITE_AT_END | + HA_STATE_EXTEND_BLOCK); + } + break; + case HA_EXTRA_PREPARE_FOR_UPDATE: + if (info->s->data_file_type != DYNAMIC_RECORD) + break; + /* Remove read/write cache if dynamic rows */ + /* fall through */ + case HA_EXTRA_NO_CACHE: + if (info->opt_flag & (READ_CACHE_USED | WRITE_CACHE_USED)) + { + info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED); + error=end_io_cache(&info->rec_cache); + /* Sergei will insert full text index caching here */ + } +#if defined(HAVE_MMAP) && defined(HAVE_MADVISE) + if (info->opt_flag & MEMMAP_USED) + madvise((char*) share->file_map, share->state.state.data_file_length, + MADV_RANDOM); +#endif + break; + case HA_EXTRA_FLUSH_CACHE: + if (info->opt_flag & WRITE_CACHE_USED) + { + if ((error=flush_io_cache(&info->rec_cache))) + { + mi_print_error(info->s, HA_ERR_CRASHED); + mi_mark_crashed(info); /* Fatal error found */ + } + } + break; + case HA_EXTRA_NO_READCHECK: + info->opt_flag&= ~READ_CHECK_USED; /* No readcheck */ + break; + case HA_EXTRA_READCHECK: + info->opt_flag|= READ_CHECK_USED; + break; + case HA_EXTRA_KEYREAD: /* Read only keys to record */ + case HA_EXTRA_REMEMBER_POS: + info->opt_flag |= REMEMBER_OLD_POS; + bmove((uchar*) info->lastkey+share->base.max_key_length*2, + (uchar*) info->lastkey,info->lastkey_length); + info->save_update= info->update; + info->save_lastinx= info->lastinx; + info->save_lastpos= info->lastpos; + info->save_lastkey_length=info->lastkey_length; + if (function == HA_EXTRA_REMEMBER_POS) + break; + /* fall through */ + case HA_EXTRA_KEYREAD_CHANGE_POS: + info->opt_flag |= KEY_READ_USED; + info->read_record=_mi_read_key_record; + break; + case HA_EXTRA_NO_KEYREAD: + case HA_EXTRA_RESTORE_POS: + if (info->opt_flag & REMEMBER_OLD_POS) + { + bmove((uchar*) info->lastkey, + (uchar*) info->lastkey+share->base.max_key_length*2, + info->save_lastkey_length); + info->update= info->save_update | HA_STATE_WRITTEN; + info->lastinx= info->save_lastinx; + info->lastpos= info->save_lastpos; + info->lastkey_length=info->save_lastkey_length; + } + info->read_record= share->read_record; + info->opt_flag&= ~(KEY_READ_USED | REMEMBER_OLD_POS); + break; + case HA_EXTRA_NO_USER_CHANGE: /* Database is somehow locked against changes */ + info->lock_type= F_EXTRA_LCK; /* Simulate as locked */ + break; + case HA_EXTRA_WAIT_LOCK: + info->lock_wait=0; + break; + case HA_EXTRA_NO_WAIT_LOCK: + info->lock_wait= MY_SHORT_WAIT; + break; + case HA_EXTRA_NO_KEYS: + if (info->lock_type == F_UNLCK) + { + error=1; /* Not possibly if not lock */ + break; + } + if (mi_is_any_key_active(share->state.key_map)) + { + MI_KEYDEF *key=share->keyinfo; + uint i; + for (i=0 ; i < share->base.keys ; i++,key++) + { + if (!(key->flag & HA_NOSAME) && info->s->base.auto_key != i+1) + { + mi_clear_key_active(share->state.key_map, i); + info->update|= HA_STATE_CHANGED; + } + } + + if (!share->changed) + { + share->state.changed|= STATE_CHANGED | STATE_NOT_ANALYZED; + share->changed=1; /* Update on close */ + if (!share->global_changed) + { + share->global_changed=1; + share->state.open_count++; + } + } + share->state.state= *info->state; + error=mi_state_info_write(share->kfile,&share->state,1 | 2); + } + break; + case HA_EXTRA_FORCE_REOPEN: + mysql_mutex_lock(&THR_LOCK_myisam); + share->last_version= 0L; /* Impossible version */ + mysql_mutex_unlock(&THR_LOCK_myisam); + break; + case HA_EXTRA_PREPARE_FOR_DROP: + /* Signals about intent to delete this table */ + share->deleting= TRUE; + share->global_changed= FALSE; /* force writing changed flag */ + _mi_mark_file_changed(info); + if (share->temporary) + break; + /* fall through */ + case HA_EXTRA_PREPARE_FOR_RENAME: + DBUG_ASSERT(!share->temporary); + mysql_mutex_lock(&THR_LOCK_myisam); + share->last_version= 0L; /* Impossible version */ + mysql_mutex_lock(&share->intern_lock); + /* Flush pages that we don't need anymore */ + if (flush_key_blocks(share->key_cache, share->kfile, + &share->dirty_part_map, + (function == HA_EXTRA_PREPARE_FOR_DROP ? + FLUSH_IGNORE_CHANGED : FLUSH_RELEASE))) + { + error=my_errno; + share->changed=1; + mi_print_error(info->s, HA_ERR_CRASHED); + mi_mark_crashed(info); /* Fatal error found */ + } + mysql_mutex_unlock(&share->intern_lock); + mysql_mutex_unlock(&THR_LOCK_myisam); + break; + case HA_EXTRA_END_ALTER_COPY: + case HA_EXTRA_FLUSH: + if (!share->temporary) + flush_key_blocks(share->key_cache, share->kfile, &share->dirty_part_map, + FLUSH_KEEP); + mysql_mutex_lock(&share->intern_lock); + /* Tell mi_lock_database() that we locked the intern_lock mutex */ + info->intern_lock_locked= 1; + _mi_decrement_open_count(info); + info->intern_lock_locked= 0; + if (share->not_flushed) + { + share->not_flushed=0; + if (mysql_file_sync(share->kfile, MYF(0))) + error= my_errno; + if (mysql_file_sync(info->dfile, MYF(0))) + error= my_errno; + if (error) + { + share->changed=1; + mi_print_error(info->s, HA_ERR_CRASHED); + mi_mark_crashed(info); /* Fatal error found */ + } + } + if (share->base.blobs) + mi_alloc_rec_buff(info, -1, &info->rec_buff); + mysql_mutex_unlock(&share->intern_lock); + break; + case HA_EXTRA_NORMAL: /* These aren't in use */ + info->quick_mode=0; + break; + case HA_EXTRA_QUICK: + info->quick_mode=1; + break; + case HA_EXTRA_NO_ROWS: + if (!share->state.header.uniques) + info->opt_flag|= OPT_NO_ROWS; + break; + case HA_EXTRA_PRELOAD_BUFFER_SIZE: + info->preload_buff_size= *((ulong *) extra_arg); + break; + case HA_EXTRA_CHANGE_KEY_TO_UNIQUE: + case HA_EXTRA_CHANGE_KEY_TO_DUP: + mi_extra_keyflag(info, function); + break; + case HA_EXTRA_MMAP: +#ifdef HAVE_MMAP + mysql_mutex_lock(&share->intern_lock); + /* + Memory map the data file if it is not already mapped. It is safe + to memory map a file while other threads are using file I/O on it. + Assigning a new address to a function pointer is an atomic + operation. intern_lock prevents that two or more mappings are done + at the same time. + */ + if (!share->file_map) + { + if (mi_dynmap_file(info, share->state.state.data_file_length)) + { + DBUG_PRINT("warning",("mmap failed: errno: %d",errno)); + error= my_errno= errno; + } + } + mysql_mutex_unlock(&share->intern_lock); +#endif + break; + case HA_EXTRA_MARK_AS_LOG_TABLE: + mysql_mutex_lock(&share->intern_lock); + share->is_log_table= TRUE; + mysql_mutex_unlock(&share->intern_lock); + break; + case HA_EXTRA_DETACH_CHILD: /* When used with MERGE tables */ + info->open_flag&= ~HA_OPEN_MERGE_TABLE; + info->lock.priority&= ~THR_LOCK_MERGE_PRIV; + break; + + case HA_EXTRA_KEY_CACHE: + case HA_EXTRA_NO_KEY_CACHE: + default: + break; + } + { + char tmp[1]; + tmp[0]=function; + myisam_log_command(MI_LOG_EXTRA,info,(uchar*) tmp,1,error); + } + DBUG_RETURN(error); +} /* mi_extra */ + +void mi_set_index_cond_func(MI_INFO *info, index_cond_func_t func, + void *func_arg) +{ + info->index_cond_func= func; + info->index_cond_func_arg= func_arg; +} + +void mi_set_rowid_filter_func(MI_INFO *info, + rowid_filter_func_t check_func, + rowid_filter_is_active_func_t is_active_func, + void *func_arg) +{ + info->rowid_filter_func= check_func; + info->rowid_filter_is_active_func= is_active_func; + info->rowid_filter_func_arg= func_arg; +} + +/* + Start/Stop Inserting Duplicates Into a Table, WL#1648. + */ +static void mi_extra_keyflag(MI_INFO *info, enum ha_extra_function function) +{ + uint idx; + + for (idx= 0; idx< info->s->base.keys; idx++) + { + switch (function) { + case HA_EXTRA_CHANGE_KEY_TO_UNIQUE: + info->s->keyinfo[idx].flag|= HA_NOSAME; + break; + case HA_EXTRA_CHANGE_KEY_TO_DUP: + info->s->keyinfo[idx].flag&= ~(HA_NOSAME); + break; + default: + break; + } + } +} + + +int mi_reset(MI_INFO *info) +{ + int error= 0; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("mi_reset"); + /* + Free buffers and reset the following flags: + EXTRA_CACHE, EXTRA_WRITE_CACHE, EXTRA_KEYREAD, EXTRA_QUICK + + If the row buffer cache is large (for dynamic tables), reduce it + to save memory. + */ + if (info->opt_flag & (READ_CACHE_USED | WRITE_CACHE_USED)) + { + info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED); + error= end_io_cache(&info->rec_cache); + } + if (share->base.blobs) + mi_alloc_rec_buff(info, -1, &info->rec_buff); +#if defined(HAVE_MMAP) && defined(HAVE_MADVISE) + if (info->opt_flag & MEMMAP_USED) + madvise((char*) share->file_map, share->state.state.data_file_length, + MADV_RANDOM); +#endif + info->opt_flag&= ~(KEY_READ_USED | REMEMBER_OLD_POS); + info->quick_mode=0; + info->lastinx= 0; /* Use first index as def */ + info->last_search_keypage= info->lastpos= HA_OFFSET_ERROR; + info->page_changed= 1; + info->update= ((info->update & HA_STATE_CHANGED) | HA_STATE_NEXT_FOUND | + HA_STATE_PREV_FOUND); + DBUG_RETURN(error); +} + +my_bool mi_killed_standalone(MI_INFO *info __attribute__((unused))) +{ + return 0; +} diff --git a/storage/myisam/mi_extrafunc.h b/storage/myisam/mi_extrafunc.h new file mode 100644 index 00000000..bfefb52c --- /dev/null +++ b/storage/myisam/mi_extrafunc.h @@ -0,0 +1,22 @@ +/* Copyright (c) 2000-2006 MySQL AB, 2009 Sun Microsystems, Inc. + Use is subject to license terms. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +void _mi_report_crashed(MI_INFO *file __attribute__((unused)), + const char *message __attribute__((unused)), + const char *sfile __attribute__((unused)), + uint sline __attribute__((unused))) +{ +} diff --git a/storage/myisam/mi_info.c b/storage/myisam/mi_info.c new file mode 100644 index 00000000..b754dde2 --- /dev/null +++ b/storage/myisam/mi_info.c @@ -0,0 +1,136 @@ +/* Copyright (c) 2000, 2001, 2003-2007 MySQL AB, 2009 Sun Microsystems, Inc. + Use is subject to license terms. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Return useful base information for an open table */ + +#include "myisamdef.h" +#ifdef _WIN32 +#include <sys/stat.h> +#endif + + /* Get position to last record */ + +my_off_t mi_position(MI_INFO *info) +{ + return info->lastpos; +} + + +/* Get information about the table */ +/* if flag == 2 one get current info (no sync from database */ + +int mi_status(MI_INFO *info, register MI_ISAMINFO *x, uint flag) +{ + MY_STAT state; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("mi_status"); + + x->recpos = info->lastpos; + if (flag == HA_STATUS_POS) + DBUG_RETURN(0); /* Compatible with ISAM */ + if (!(flag & HA_STATUS_NO_LOCK)) + { + mysql_mutex_lock(&share->intern_lock); + (void) _mi_readinfo(info,F_RDLCK,0); + fast_mi_writeinfo(info); + mysql_mutex_unlock(&share->intern_lock); + } + if (flag & HA_STATUS_VARIABLE) + { + x->records = info->state->records; + x->deleted = info->state->del; + x->delete_length = info->state->empty; + x->data_file_length =info->state->data_file_length; + x->index_file_length=info->state->key_file_length; + + x->keys = share->state.header.keys; + x->check_time = share->state.check_time; + x->mean_reclength= x->records ? + (ulong) ((x->data_file_length - x->delete_length) / x->records) : + (ulong) share->min_pack_length; + } + if (flag & HA_STATUS_ERRKEY) + { + x->errkey = info->errkey; + x->dupp_key_pos= info->dupp_key_pos; + } + if (flag & HA_STATUS_CONST) + { + x->reclength = share->base.reclength; + x->max_data_file_length=share->base.max_data_file_length; + x->max_index_file_length=info->s->base.max_key_file_length; + x->filenr = info->dfile; + x->options = share->options; + x->create_time=share->state.create_time; + x->reflength= mi_get_pointer_length(share->base.max_data_file_length, + myisam_data_pointer_size); + x->record_offset= ((share->options & + (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ? + 0L : share->base.pack_reclength); + x->sortkey= -1; /* No clustering */ + x->rec_per_key = share->state.rec_per_key_part; + x->key_map = share->state.key_map; + x->data_file_name = share->data_file_name; + x->index_file_name = share->index_file_name; + } + if ((flag & HA_STATUS_TIME) && !mysql_file_fstat(info->dfile, &state, MYF(0))) + { + MSAN_STAT_WORKAROUND(&state); + x->update_time=state.st_mtime; + } + else + x->update_time=0; + if (flag & HA_STATUS_AUTO) + { + x->auto_increment= share->state.auto_increment+1; + if (!x->auto_increment) /* This shouldn't happen */ + x->auto_increment= ~(ulonglong) 0; + } + DBUG_RETURN(0); +} + + +/* + Write a message to the error log. + + SYNOPSIS + mi_report_error() + file_name Name of table file (e.g. index_file_name). + errcode Error number. + + DESCRIPTION + This function supplies my_error() with a table name. Most error + messages need one. Since string arguments in error messages are limited + to 64 characters by convention, we ensure that in case of truncation, + that the end of the index file path is in the message. This contains + the most valuable information (the table name and the database name). + + RETURN + void +*/ + +void mi_report_error(int errcode, const char *file_name) +{ + size_t lgt; + DBUG_ENTER("mi_report_error"); + DBUG_PRINT("enter",("errcode %d, table '%s'", errcode, file_name)); + + if ((lgt= strlen(file_name)) > 64) + file_name+= lgt - 64; + my_error(errcode, MYF(ME_ERROR_LOG), file_name); + DBUG_VOID_RETURN; +} + diff --git a/storage/myisam/mi_key.c b/storage/myisam/mi_key.c new file mode 100644 index 00000000..087eb59c --- /dev/null +++ b/storage/myisam/mi_key.c @@ -0,0 +1,662 @@ +/* Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2020, MariaDB Corporation. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Functions to handle keys */ + +#include "myisamdef.h" +#include "m_ctype.h" +#include "sp_defs.h" +#ifdef HAVE_IEEEFP_H +#include <ieeefp.h> +#endif + +#define CHECK_KEYS /* Enable safety checks */ + +#define FIX_LENGTH(cs, pos, length, char_length) \ + do { \ + if (length > char_length) \ + char_length= my_ci_charpos(cs, (const char *) pos, \ + (const char *) pos+length, \ + char_length); \ + set_if_smaller(char_length,length); \ + } while(0) + +static int _mi_put_key_in_record(MI_INFO *info,uint keynr, + my_bool unpack_blobs, uchar *record); + +/* + Make a intern key from a record + + SYNOPSIS + _mi_make_key() + info MyiSAM handler + keynr key number + key Store created key here + record Record + filepos Position to record in the data file + + RETURN + Length of key +*/ + +uint _mi_make_key(register MI_INFO *info, uint keynr, uchar *key, + const uchar *record, my_off_t filepos) +{ + uchar *pos; + uchar *start; + reg1 HA_KEYSEG *keyseg; + my_bool is_ft= info->s->keyinfo[keynr].flag & HA_FULLTEXT; + DBUG_ENTER("_mi_make_key"); + + if (info->s->keyinfo[keynr].flag & HA_SPATIAL) + { + /* + TODO: nulls processing + */ +#ifdef HAVE_SPATIAL + DBUG_RETURN(sp_make_key(info,keynr,key,record,filepos)); +#else + DBUG_ASSERT(0); /* mi_open should check that this never happens*/ +#endif + } + + start=key; + for (keyseg=info->s->keyinfo[keynr].seg ; keyseg->type ;keyseg++) + { + enum ha_base_keytype type=(enum ha_base_keytype) keyseg->type; + size_t length=keyseg->length; + size_t char_length; + CHARSET_INFO *cs=keyseg->charset; + + if (keyseg->null_bit) + { + if (record[keyseg->null_pos] & keyseg->null_bit) + { + *key++= 0; /* NULL in key */ + continue; + } + *key++=1; /* Not NULL */ + } + + char_length= ((!is_ft && cs && cs->mbmaxlen > 1) ? length/cs->mbmaxlen : + length); + + pos= (uchar*) record+keyseg->start; + if (type == HA_KEYTYPE_BIT) + { + if (keyseg->bit_length) + { + uchar bits= get_rec_bits((uchar*) record + keyseg->bit_pos, + keyseg->bit_start, keyseg->bit_length); + *key++= bits; + length--; + } + memcpy((uchar*) key, pos, length); + key+= length; + continue; + } + if (keyseg->flag & HA_SPACE_PACK) + { + if (type != HA_KEYTYPE_NUM) + { + length= my_ci_lengthsp(cs, (char*) pos, length); + } + else + { + uchar *end= pos + length; + while (pos < end && pos[0] == ' ') + pos++; + length=(size_t) (end-pos); + } + FIX_LENGTH(cs, pos, length, char_length); + store_key_length_inc(key,char_length); + memcpy(key, pos,char_length); + key+=char_length; + continue; + } + if (keyseg->flag & HA_VAR_LENGTH_PART) + { + uint pack_length= (keyseg->bit_start == 1 ? 1 : 2); + uint tmp_length= (pack_length == 1 ? (uint) *(uchar*) pos : + uint2korr(pos)); + pos+= pack_length; /* Skip VARCHAR length */ + set_if_smaller(length,tmp_length); + FIX_LENGTH(cs, pos, length, char_length); + store_key_length_inc(key,char_length); + memcpy(key, pos, char_length); + key+= char_length; + continue; + } + else if (keyseg->flag & HA_BLOB_PART) + { + uint tmp_length=_mi_calc_blob_length(keyseg->bit_start,pos); + memcpy(&pos,pos+keyseg->bit_start,sizeof(char*)); + set_if_smaller(length,tmp_length); + FIX_LENGTH(cs, pos, length, char_length); + store_key_length_inc(key,char_length); + if (char_length) + { + memcpy(key, pos, char_length); + key+= char_length; + } + continue; + } + else if (keyseg->flag & HA_SWAP_KEY) + { /* Numerical column */ + if (type == HA_KEYTYPE_FLOAT) + { + float nr; + float4get(nr,pos); + if (isnan(nr)) + { + /* Replace NAN with zero */ + bzero(key,length); + key+=length; + continue; + } + } + else if (type == HA_KEYTYPE_DOUBLE) + { + double nr; + float8get(nr,pos); + if (isnan(nr)) + { + bzero(key,length); + key+=length; + continue; + } + } + pos+=length; + while (length--) + { + *key++ = *--pos; + } + continue; + } + FIX_LENGTH(cs, pos, length, char_length); + memcpy((uchar*) key, pos, char_length); + if (length > char_length) + my_ci_fill(cs, (char*) key+char_length, length-char_length, ' '); + key+= length; + } + _mi_dpointer(info,key,filepos); + DBUG_PRINT("exit",("keynr: %d",keynr)); + DBUG_DUMP("key",(uchar*) start,(uint) (key-start)+keyseg->length); + DBUG_EXECUTE("key", + _mi_print_key(DBUG_FILE,info->s->keyinfo[keynr].seg,start, + (uint) (key-start));); + DBUG_RETURN((uint) (key-start)); /* Return keylength */ +} /* _mi_make_key */ + + +/* + Pack a key to intern format from given format (c_rkey) + + SYNOPSIS + _mi_pack_key() + info MyISAM handler + uint keynr key number + key Store packed key here + old Not packed key + keypart_map bitmap of used keyparts + last_used_keyseg out parameter. May be NULL + + RETURN + length of packed key + + last_use_keyseg Store pointer to the keyseg after the last used one +*/ + +uint _mi_pack_key(register MI_INFO *info, uint keynr, uchar *key, uchar *old, + key_part_map keypart_map, HA_KEYSEG **last_used_keyseg) +{ + uchar *start_key=key; + HA_KEYSEG *keyseg; + my_bool is_ft= info->s->keyinfo[keynr].flag & HA_FULLTEXT; + DBUG_ENTER("_mi_pack_key"); + + /* "one part" rtree key is 2*SPDIMS part key in MyISAM */ + if (info->s->keyinfo[keynr].key_alg == HA_KEY_ALG_RTREE) + keypart_map= (((key_part_map)1) << (2*SPDIMS)) - 1; + + /* only key prefixes are supported */ + DBUG_ASSERT(((keypart_map+1) & keypart_map) == 0); + + for (keyseg= info->s->keyinfo[keynr].seg ; keyseg->type && keypart_map; + old+= keyseg->length, keyseg++) + { + enum ha_base_keytype type= (enum ha_base_keytype) keyseg->type; + size_t length= keyseg->length; + size_t char_length; + uchar *pos; + CHARSET_INFO *cs=keyseg->charset; + + keypart_map>>= 1; + if (keyseg->null_bit) + { + if (!(*key++= (char) 1-*old++)) /* Copy null marker */ + { + if (keyseg->flag & (HA_VAR_LENGTH_PART | HA_BLOB_PART)) + old+= 2; + continue; /* Found NULL */ + } + } + char_length= (!is_ft && cs && cs->mbmaxlen > 1) ? length/cs->mbmaxlen : length; + pos=old; + if (keyseg->flag & HA_SPACE_PACK) + { + if (type == HA_KEYTYPE_NUM) + { + uchar *end= pos + length; + while (pos < end && pos[0] == ' ') + pos++; + length= (size_t)(end - pos); + } + else if (type != HA_KEYTYPE_BINARY) + { + length= my_ci_lengthsp(cs, (char*) pos, length); + } + FIX_LENGTH(cs, pos, length, char_length); + store_key_length_inc(key,char_length); + memcpy(key,pos,char_length); + key+= char_length; + continue; + } + else if (keyseg->flag & (HA_VAR_LENGTH_PART | HA_BLOB_PART)) + { + /* Length of key-part used with mi_rkey() always 2 */ + uint tmp_length=uint2korr(pos); + pos+=2; + set_if_smaller(length,tmp_length); /* Safety */ + FIX_LENGTH(cs, pos, length, char_length); + store_key_length_inc(key,char_length); + old+=2; /* Skip length */ + memcpy(key, pos, char_length); + key+= char_length; + continue; + } + else if (keyseg->flag & HA_SWAP_KEY) + { /* Numerical column */ + pos+=length; + while (length--) + *key++ = *--pos; + continue; + } + FIX_LENGTH(cs, pos, length, char_length); + memcpy((uchar*) key, pos, char_length); + if (length > char_length) + my_ci_fill(cs, (char*) key+char_length, length-char_length, ' '); + key+= length; + } + if (last_used_keyseg) + *last_used_keyseg= keyseg; + + DBUG_RETURN((uint) (key-start_key)); +} /* _mi_pack_key */ + + + +/* + Store found key in record + + SYNOPSIS + _mi_put_key_in_record() + info MyISAM handler + keynr Key number that was used + unpack_blobs TRUE <=> Unpack blob columns + FALSE <=> Skip them. This is used by index condition + pushdown check function + record Store key here + + Last read key is in info->lastkey + + NOTES + Used when only-keyread is wanted + + RETURN + 0 ok + 1 error +*/ + +static int _mi_put_key_in_record(register MI_INFO *info, uint keynr, + my_bool unpack_blobs, uchar *record) +{ + reg2 uchar *key; + uchar *pos,*key_end; + reg1 HA_KEYSEG *keyseg; + uchar *blob_ptr; + DBUG_ENTER("_mi_put_key_in_record"); + + blob_ptr= (uchar*) info->lastkey2; /* Place to put blob parts */ + key=(uchar*) info->lastkey; /* KEy that was read */ + key_end=key+info->lastkey_length; + for (keyseg=info->s->keyinfo[keynr].seg ; keyseg->type ;keyseg++) + { + if (keyseg->null_bit) + { + if (!*key++) + { + record[keyseg->null_pos]|= keyseg->null_bit; + continue; + } + record[keyseg->null_pos]&= ~keyseg->null_bit; + } + if (keyseg->type == HA_KEYTYPE_BIT) + { + uint length= keyseg->length; + + if (keyseg->bit_length) + { + uchar bits= *key++; + set_rec_bits(bits, record + keyseg->bit_pos, keyseg->bit_start, + keyseg->bit_length); + length--; + } + else + { + clr_rec_bits(record + keyseg->bit_pos, keyseg->bit_start, + keyseg->bit_length); + } + memcpy(record + keyseg->start, (uchar*) key, length); + key+= length; + continue; + } + if (keyseg->flag & HA_SPACE_PACK) + { + uint length; + get_key_length(length,key); +#ifdef CHECK_KEYS + if (length > keyseg->length || key+length > key_end) + goto err; +#endif + pos= record+keyseg->start; + if (keyseg->type != (int) HA_KEYTYPE_NUM) + { + memcpy(pos,key,(size_t) length); + my_ci_fill(keyseg->charset, (char*) pos + length, + keyseg->length - length, + ' '); + } + else + { + bfill(pos,keyseg->length-length,' '); + memcpy(pos+keyseg->length-length,key,(size_t) length); + } + key+=length; + continue; + } + + if (keyseg->flag & HA_VAR_LENGTH_PART) + { + uint length; + get_key_length(length,key); +#ifdef CHECK_KEYS + if (length > keyseg->length || key+length > key_end) + goto err; +#endif + /* Store key length */ + if (keyseg->bit_start == 1) + *(uchar*) (record+keyseg->start)= (uchar) length; + else + int2store(record+keyseg->start, length); + /* And key data */ + memcpy(record+keyseg->start + keyseg->bit_start, (uchar*) key, length); + key+= length; + } + else if (keyseg->flag & HA_BLOB_PART) + { + uint length; + get_key_length(length,key); +#ifdef CHECK_KEYS + if (length > keyseg->length || key+length > key_end) + goto err; +#endif + if (unpack_blobs) + { + memcpy(record+keyseg->start+keyseg->bit_start, + &blob_ptr, sizeof(char *)); + memcpy(blob_ptr,key,length); + blob_ptr+=length; + + /* The above changed info->lastkey2. Inform mi_rnext_same(). */ + info->update&= ~HA_STATE_RNEXT_SAME; + + _mi_store_blob_length(record+keyseg->start, + (uint) keyseg->bit_start,length); + } + key+=length; + } + else if (keyseg->flag & HA_SWAP_KEY) + { + uchar *to= record+keyseg->start+keyseg->length; + uchar *end= key+keyseg->length; +#ifdef CHECK_KEYS + if (end > key_end) + goto err; +#endif + do + { + *--to= *key++; + } while (key != end); + continue; + } + else + { +#ifdef CHECK_KEYS + if (key+keyseg->length > key_end) + goto err; +#endif + memcpy(record+keyseg->start,(uchar*) key, + (size_t) keyseg->length); + key+= keyseg->length; + } + } + DBUG_RETURN(0); + +err: + DBUG_RETURN(1); /* Crashed row */ +} /* _mi_put_key_in_record */ + + + /* Here when key reads are used */ + +int _mi_read_key_record(MI_INFO *info, my_off_t filepos, uchar *buf) +{ + fast_mi_writeinfo(info); + if (filepos != HA_OFFSET_ERROR) + { + if (info->lastinx >= 0) + { /* Read only key */ + if (_mi_put_key_in_record(info,(uint) info->lastinx, TRUE, buf)) + { + mi_print_error(info->s, HA_ERR_CRASHED); + my_errno=HA_ERR_CRASHED; + return -1; + } + info->update|= HA_STATE_AKTIV; /* We should find a record */ + return 0; + } + my_errno=HA_ERR_WRONG_INDEX; + } + return(-1); /* Wrong data to read */ +} + + +static +int mi_unpack_index_tuple(MI_INFO *info, uint keynr, uchar *record) +{ + if (_mi_put_key_in_record(info, keynr, FALSE, record)) + { + /* Impossible case; Can only happen if bug in code */ + mi_print_error(info->s, HA_ERR_CRASHED); + info->lastpos= HA_OFFSET_ERROR; /* No active record */ + my_errno= HA_ERR_CRASHED; + return 1; + } + return 0; +} + + +static int mi_check_rowid_filter_is_active(MI_INFO *info) +{ + if (info->rowid_filter_is_active_func == NULL) + return 0; + return info->rowid_filter_is_active_func(info->rowid_filter_func_arg); +} + + +/* + Check the current index tuple: Check ICP condition and/or Rowid Filter + + SYNOPSIS + mi_check_index_tuple() + info MyISAM handler + keynr Index we're running a scan on + record Record buffer to use (it is assumed that index check function + will look for column values there) + + RETURN + Check result according to check_result_t definition +*/ + +check_result_t mi_check_index_tuple(MI_INFO *info, uint keynr, uchar *record) +{ + int need_unpack= TRUE; + check_result_t res= CHECK_POS; + + if (info->index_cond_func) + { + if (mi_unpack_index_tuple(info, keynr, record)) + res= CHECK_ERROR; + else if ((res= info->index_cond_func(info->index_cond_func_arg)) == + CHECK_OUT_OF_RANGE) + { + /* We got beyond the end of scanned range */ + info->lastpos= HA_OFFSET_ERROR; /* No active record */ + my_errno= HA_ERR_END_OF_FILE; + } + + /* + If we got an error, out-of-range condition, or ICP condition computed to + FALSE - we don't need to check the Rowid Filter. + */ + if (res != CHECK_POS) + return res; + + need_unpack= FALSE; + } + + /* Check the Rowid Filter, if present */ + if (mi_check_rowid_filter_is_active(info)) + { + /* Unpack the index tuple if we haven't done it already */ + if (need_unpack && mi_unpack_index_tuple(info, keynr, record)) + res= CHECK_ERROR; + else + { + if ((res= info->rowid_filter_func(info->rowid_filter_func_arg)) == + CHECK_OUT_OF_RANGE) + { + /* We got beyond the end of scanned range */ + info->lastpos= HA_OFFSET_ERROR; /* No active record */ + my_errno= HA_ERR_END_OF_FILE; + } + } + } + return res; +} + + +/* + Retrieve auto_increment info + + SYNOPSIS + retrieve_auto_increment() + info MyISAM handler + record Row to update + + IMPLEMENTATION + For signed columns we don't retrieve the auto increment value if it's + less than zero. +*/ + +ulonglong retrieve_auto_increment(MI_INFO *info,const uchar *record) +{ + ulonglong value= 0; /* Store unsigned values here */ + longlong s_value= 0; /* Store signed values here */ + HA_KEYSEG *keyseg= info->s->keyinfo[info->s->base.auto_key-1].seg; + const uchar *key= (uchar*) record + keyseg->start; + + switch (keyseg->type) { + case HA_KEYTYPE_INT8: + s_value= (longlong) *(const signed char*) key; + break; + case HA_KEYTYPE_BINARY: + value=(ulonglong) *(uchar*) key; + break; + case HA_KEYTYPE_SHORT_INT: + s_value= (longlong) sint2korr(key); + break; + case HA_KEYTYPE_USHORT_INT: + value=(ulonglong) uint2korr(key); + break; + case HA_KEYTYPE_LONG_INT: + s_value= (longlong) sint4korr(key); + break; + case HA_KEYTYPE_ULONG_INT: + value=(ulonglong) uint4korr(key); + break; + case HA_KEYTYPE_INT24: + s_value= (longlong) sint3korr(key); + break; + case HA_KEYTYPE_UINT24: + value=(ulonglong) uint3korr(key); + break; + case HA_KEYTYPE_FLOAT: /* This shouldn't be used */ + { + float f_1; + float4get(f_1,key); + /* Ignore negative values */ + value = (f_1 < (float) 0.0) ? 0 : (ulonglong) f_1; + break; + } + case HA_KEYTYPE_DOUBLE: /* This shouldn't be used */ + { + double f_1; + float8get(f_1,key); + /* Ignore negative values */ + value = (f_1 < 0.0) ? 0 : (ulonglong) f_1; + break; + } + case HA_KEYTYPE_LONGLONG: + s_value= sint8korr(key); + break; + case HA_KEYTYPE_ULONGLONG: + value= uint8korr(key); + break; + default: + DBUG_ASSERT(0); + value=0; /* Error */ + break; + } + + /* + The following code works because if s_value < 0 then value is 0 + and if s_value == 0 then value will contain either s_value or the + correct value. + */ + return (s_value > 0) ? (ulonglong) s_value : value; +} diff --git a/storage/myisam/mi_keycache.c b/storage/myisam/mi_keycache.c new file mode 100644 index 00000000..d7f1d8b1 --- /dev/null +++ b/storage/myisam/mi_keycache.c @@ -0,0 +1,172 @@ +/* Copyright (c) 2003-2008 MySQL AB, 2009 Sun Microsystems, Inc. + Use is subject to license terms. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* + Key cache assignments +*/ + +#include "myisamdef.h" + +/* + Assign pages of the index file for a table to a key cache + + SYNOPSIS + mi_assign_to_key_cache() + info open table + key_map map of indexes to assign to the key cache + key_cache_ptr pointer to the key cache handle + assign_lock Mutex to lock during assignment + + PREREQUESTS + One must have a READ lock or a WRITE lock on the table when calling + the function to ensure that there is no other writers to it. + + The caller must also ensure that one doesn't call this function from + two different threads with the same table. + + NOTES + At present pages for all indexes must be assigned to the same key cache. + In future only pages for indexes specified in the key_map parameter + of the table will be assigned to the specified key cache. + + RETURN VALUE + 0 If a success + # Error code +*/ + +int mi_assign_to_key_cache(MI_INFO *info, + ulonglong key_map __attribute__((unused)), + KEY_CACHE *new_key_cache) +{ + int error= 0; + MYISAM_SHARE* share= info->s; + KEY_CACHE *old_key_cache= share->key_cache; + DBUG_ENTER("mi_assign_to_key_cache"); + DBUG_PRINT("enter",("old_key_cache_handle: %p new_key_cache_handle: %p", + old_key_cache, new_key_cache)); + + /* + Skip operation if we didn't change key cache. This can happen if we + call this for all open instances of the same table + */ + if (old_key_cache == new_key_cache) + DBUG_RETURN(0); + + /* + First flush all blocks for the table in the old key cache. + This is to ensure that the disk is consistent with the data pages + in memory (which may not be the case if the table uses delayed_key_write) + + Note that some other read thread may still fill in the key cache with + new blocks during this call and after, but this doesn't matter as + all threads will start using the new key cache for their next call to + myisam library and we know that there will not be any changed blocks + in the old key cache. + */ + + pthread_mutex_lock(&old_key_cache->op_lock); + DEBUG_SYNC_C("assign_key_cache_op_lock"); + if (flush_key_blocks(old_key_cache, share->kfile, &share->dirty_part_map, + FLUSH_RELEASE)) + { + error= my_errno; + mi_print_error(info->s, HA_ERR_CRASHED); + mi_mark_crashed(info); /* Mark that table must be checked */ + } + pthread_mutex_unlock(&old_key_cache->op_lock); + DEBUG_SYNC_C("assign_key_cache_op_unlock"); + + /* + Flush the new key cache for this file. This is needed to ensure + that there is no old blocks (with outdated data) left in the new key + cache from an earlier assign_to_keycache operation + + (This can never fail as there is never any not written data in the + new key cache) + */ + (void) flush_key_blocks(new_key_cache, share->kfile, &share->dirty_part_map, + FLUSH_RELEASE); + + /* + ensure that setting the key cache and changing the multi_key_cache + is done atomicly + */ + mysql_mutex_lock(&share->intern_lock); + /* + Tell all threads to use the new key cache + This should be seen at the lastes for the next call to an myisam function. + */ + share->key_cache= new_key_cache; + share->dirty_part_map= 0; + + /* store the key cache in the global hash structure for future opens */ + if (multi_key_cache_set((uchar*) share->unique_file_name, + share->unique_name_length, + new_key_cache)) + error= my_errno; + mysql_mutex_unlock(&share->intern_lock); + DBUG_RETURN(error); +} + + +/* + Change all MyISAM entries that uses one key cache to another key cache + + SYNOPSIS + mi_change_key_cache() + old_key_cache Old key cache + new_key_cache New key cache + + NOTES + This is used when we delete one key cache. + + To handle the case where some other threads tries to open an MyISAM + table associated with the to-be-deleted key cache while this operation + is running, we have to call 'multi_key_cache_change()' from this + function while we have a lock on the MyISAM table list structure. + + This is safe as long as it's only MyISAM that is using this specific + key cache. +*/ + + +void mi_change_key_cache(KEY_CACHE *old_key_cache, + KEY_CACHE *new_key_cache) +{ + LIST *pos; + DBUG_ENTER("mi_change_key_cache"); + + /* + Lock list to ensure that no one can close the table while we manipulate it + */ + mysql_mutex_lock(&THR_LOCK_myisam); + for (pos=myisam_open_list ; pos ; pos=pos->next) + { + MI_INFO *info= (MI_INFO*) pos->data; + MYISAM_SHARE *share= info->s; + if (share->key_cache == old_key_cache) + mi_assign_to_key_cache(info, (ulonglong) ~0, new_key_cache); + } + + /* + We have to do the following call while we have the lock on the + MyISAM list structure to ensure that another thread is not trying to + open a new table that will be associted with the old key cache + */ + multi_key_cache_change(old_key_cache, new_key_cache); + mysql_mutex_unlock(&THR_LOCK_myisam); + DBUG_VOID_RETURN; +} diff --git a/storage/myisam/mi_locking.c b/storage/myisam/mi_locking.c new file mode 100644 index 00000000..cee1c326 --- /dev/null +++ b/storage/myisam/mi_locking.c @@ -0,0 +1,669 @@ +/* Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2009, 2018, MariaDB Corporation + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* + locking of isam-tables. + reads info from a isam-table. Must be first request before doing any furter + calls to any isamfunktion. Is used to allow many process use the same + isamdatabase. +*/ + +#include "ftdefs.h" + +static void mi_update_status_with_lock(MI_INFO *info); + + /* lock table by F_UNLCK, F_RDLCK or F_WRLCK */ + +int mi_lock_database(MI_INFO *info, int lock_type) +{ + int error, mark_crashed= 0; + uint count; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("mi_lock_database"); + DBUG_PRINT("enter",("lock_type: %d old lock %d r_locks: %u w_locks: %u " + "global_changed: %d open_count: %u name: '%s'", + lock_type, info->lock_type, share->r_locks, + share->w_locks, + share->global_changed, share->state.open_count, + share->index_file_name)); + if (share->options & HA_OPTION_READ_ONLY_DATA || + info->lock_type == lock_type) + DBUG_RETURN(0); + if (lock_type == F_EXTRA_LCK) /* Used by TMP tables */ + { + ++share->w_locks; + ++share->tot_locks; + info->lock_type= lock_type; + info->s->in_use= list_add(info->s->in_use, &info->in_use); + DBUG_RETURN(0); + } + + error= 0; + DBUG_EXECUTE_IF ("mi_lock_database_failure", error= EINVAL;); + if (!info->intern_lock_locked) + mysql_mutex_lock(&share->intern_lock); + if (share->kfile >= 0) /* May only be false on windows */ + { + switch (lock_type) { + case F_UNLCK: + ftparser_call_deinitializer(info); + if (info->lock_type == F_RDLCK) + { + count= --share->r_locks; + mi_restore_status(info); + } + else + { + count= --share->w_locks; + mi_update_status_with_lock(info); + } + --share->tot_locks; + if (info->lock_type == F_WRLCK && !share->w_locks && + !share->delay_key_write && flush_key_blocks(share->key_cache, + share->kfile, + &share->dirty_part_map, + FLUSH_KEEP)) + { + mark_crashed= error=my_errno; + mi_print_error(info->s, HA_ERR_CRASHED); + } + if (info->opt_flag & (READ_CACHE_USED | WRITE_CACHE_USED)) + { + if (end_io_cache(&info->rec_cache)) + { + mark_crashed= error=my_errno; + mi_print_error(info->s, HA_ERR_CRASHED); + } + } + if (!count) + { + DBUG_PRINT("info",("changed: %u w_locks: %u", + (uint) share->changed, share->w_locks)); + if (share->changed && !share->w_locks) + { +#ifdef HAVE_MMAP + if ((info->s->mmaped_length != info->s->state.state.data_file_length) && + (info->s->nonmmaped_inserts > MAX_NONMAPPED_INSERTS)) + { + if (info->s->concurrent_insert) + mysql_rwlock_wrlock(&info->s->mmap_lock); + mi_remap_file(info, info->s->state.state.data_file_length); + info->s->nonmmaped_inserts= 0; + if (info->s->concurrent_insert) + mysql_rwlock_unlock(&info->s->mmap_lock); + } +#endif + share->state.process= share->last_process=share->this_process; + share->state.unique= info->last_unique= info->this_unique; + share->state.update_count= info->last_loop= ++info->this_loop; + if (mi_state_info_write(share->kfile, &share->state, 1)) + mark_crashed= error=my_errno; + share->changed=0; + if (myisam_flush) + { + if (share->file_map) + my_msync(info->dfile, share->file_map, share->mmaped_length, MS_SYNC); + if (mysql_file_sync(share->kfile, MYF(0))) + mark_crashed= error= my_errno; + if (mysql_file_sync(info->dfile, MYF(0))) + mark_crashed= error= my_errno; + } + else + share->not_flushed=1; + if (error) + mi_print_error(info->s, HA_ERR_CRASHED); + } + if (info->lock_type != F_EXTRA_LCK) + { + if (share->r_locks) + { /* Only read locks left */ + if (my_lock(share->kfile,F_RDLCK,0L,F_TO_EOF, + MYF(MY_WME | MY_SEEK_NOT_DONE)) && !error) + error=my_errno; + } + else if (!share->w_locks) + { /* No more locks */ + if (my_lock(share->kfile,F_UNLCK,0L,F_TO_EOF, + MYF(MY_WME | MY_SEEK_NOT_DONE)) && !error) + error=my_errno; + } + } + } + info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED); + info->lock_type= F_UNLCK; + info->s->in_use= list_delete(info->s->in_use, &info->in_use); + break; + case F_RDLCK: + if (info->lock_type == F_WRLCK) + { + /* + Change RW to READONLY + + mysqld does not turn write locks to read locks, + so we're never here in mysqld. + */ + if (share->w_locks == 1) + { + if (my_lock(share->kfile,lock_type,0L,F_TO_EOF, + MYF(MY_SEEK_NOT_DONE))) + { + error=my_errno; + break; + } + } + share->w_locks--; + share->r_locks++; + info->lock_type=lock_type; + break; + } + if (!share->r_locks && !share->w_locks) + { + if (my_lock(share->kfile,lock_type,0L,F_TO_EOF, + info->lock_wait | MY_SEEK_NOT_DONE)) + { + error=my_errno; + break; + } + if (mi_state_info_read_dsk(share->kfile, &share->state, 1)) + { + error=my_errno; + (void) my_lock(share->kfile,F_UNLCK,0L,F_TO_EOF,MYF(MY_SEEK_NOT_DONE)); + my_errno=error; + break; + } + } + (void) _mi_test_if_changed(info); + share->r_locks++; + share->tot_locks++; + info->lock_type=lock_type; + info->s->in_use= list_add(info->s->in_use, &info->in_use); + break; + case F_WRLCK: + if (info->lock_type == F_RDLCK) + { /* Change READONLY to RW */ + if (share->r_locks == 1) + { + if (my_lock(share->kfile,lock_type,0L,F_TO_EOF, + MYF(info->lock_wait | MY_SEEK_NOT_DONE))) + { + error=my_errno; + break; + } + share->r_locks--; + share->w_locks++; + info->lock_type=lock_type; + break; + } + } + if (!(share->options & HA_OPTION_READ_ONLY_DATA)) + { + if (!share->w_locks) + { + if (my_lock(share->kfile,lock_type,0L,F_TO_EOF, + info->lock_wait | MY_SEEK_NOT_DONE)) + { + error=my_errno; + break; + } + if (!share->r_locks) + { + if (mi_state_info_read_dsk(share->kfile, &share->state, 1)) + { + error=my_errno; + (void) my_lock(share->kfile,F_UNLCK,0L,F_TO_EOF, + info->lock_wait | MY_SEEK_NOT_DONE); + my_errno=error; + break; + } + } + } + } + (void) _mi_test_if_changed(info); + + info->lock_type=lock_type; + info->invalidator=info->s->invalidator; + share->w_locks++; + share->tot_locks++; + + DBUG_EXECUTE_IF("simulate_incorrect_share_wlock_value", + DEBUG_SYNC_C("after_share_wlock_increment");); + + info->s->in_use= list_add(info->s->in_use, &info->in_use); + break; + default: + break; /* Impossible */ + } + } +#ifdef _WIN32 + else + { + /* + Check for bad file descriptors if this table is part + of a merge union. Failing to capture this may cause + a crash on windows if the table is renamed and + later on referenced by the merge table. + */ + if ((info->open_flag & HA_OPEN_MERGE_TABLE) && (info->s)->kfile < 0) + { + error = HA_ERR_NO_SUCH_TABLE; + } + } +#endif + if (!info->intern_lock_locked) + mysql_mutex_unlock(&share->intern_lock); + if (mark_crashed) + mi_mark_crashed(info); + DBUG_RETURN(error); +} /* mi_lock_database */ + + +/**************************************************************************** + The following functions are called by thr_lock() in threaded applications +****************************************************************************/ + +/* + Create a copy of the current status for the table + + SYNOPSIS + mi_get_status() + param Pointer to Myisam handler + concurrent_insert Set to 1 if we are going to do concurrent inserts + (THR_WRITE_CONCURRENT_INSERT was used) +*/ + +my_bool mi_get_status(void* param, my_bool concurrent_insert) +{ + MI_INFO *info=(MI_INFO*) param; + DBUG_ENTER("mi_get_status"); + DBUG_PRINT("info",("name: %s key_file: %lu data_file: %lu rows: %lu concurrent_insert: %d", + info->s->index_file_name, + (ulong) info->s->state.state.key_file_length, + (ulong) info->s->state.state.data_file_length, + (ulong) info->s->state.state.records, + concurrent_insert)); +#ifndef DBUG_OFF + if (info->state->key_file_length > info->s->state.state.key_file_length || + info->state->data_file_length > info->s->state.state.data_file_length) + DBUG_PRINT("warning",("old info: key_file: %ld data_file: %ld", + (long) info->state->key_file_length, + (long) info->state->data_file_length)); +#endif + info->save_state=info->s->state.state; + info->state= &info->save_state; + info->append_insert_at_end= concurrent_insert; + if (concurrent_insert) + info->s->state.state.uncacheable= TRUE; + DBUG_RETURN(0); +} + + +void mi_update_status(void* param) +{ + MI_INFO *info=(MI_INFO*) param; + DBUG_ENTER("mi_update_status"); + /* + Because someone may have closed the table we point at, we only + update the state if its our own state. This isn't a problem as + we are always pointing at our own lock or at a read lock. + (This is enforced by thr_multi_lock.c) + */ + if (info->state == &info->save_state) + { + DBUG_PRINT("info", + ("updating status: key_file: %lu data_file: %lu rows: %lu", + (ulong) info->state->key_file_length, + (ulong) info->state->data_file_length, + (ulong) info->state->records)); + if (info->state->key_file_length < info->s->state.state.key_file_length || + info->state->data_file_length < info->s->state.state.data_file_length) + DBUG_PRINT("warning",("old info: key_file: %ld data_file: %ld", + (long) info->s->state.state.key_file_length, + (long) info->s->state.state.data_file_length)); + info->s->state.state= *info->state; +#ifdef HAVE_QUERY_CACHE + DBUG_PRINT("info", ("invalidator... '%s' (status update)", + info->filename)); + DBUG_ASSERT(info->s->chst_invalidator != NULL); + (*info->s->chst_invalidator)((const char *)info->filename); +#endif + } + + info->state= &info->s->state.state; + info->append_insert_at_end= 0; + + /* + We have to flush the write cache here as other threads may start + reading the table before mi_lock_database() is called + */ + if (info->opt_flag & WRITE_CACHE_USED) + { + if (end_io_cache(&info->rec_cache)) + { + mi_print_error(info->s, HA_ERR_CRASHED); + mi_mark_crashed(info); + } + info->opt_flag&= ~WRITE_CACHE_USED; + } + DBUG_VOID_RETURN; +} + +/* + Same as mi_update_status() but take a lock in the table lock, to protect + against someone calling mi_get_status() from thr_lock() at the same time. +*/ + +static void mi_update_status_with_lock(MI_INFO *info) +{ + my_bool locked= 0; + if (info->state == &info->save_state) + { + locked= 1; + mysql_mutex_lock(&info->s->lock.mutex); + } + mi_update_status(info); + if (locked) + mysql_mutex_unlock(&info->s->lock.mutex); +} + + +void mi_restore_status(void *param) +{ + MI_INFO *info= (MI_INFO*) param; + DBUG_ENTER("mi_restore_status"); + DBUG_PRINT("info",("key_file: %ld data_file: %ld", + (long) info->s->state.state.key_file_length, + (long) info->s->state.state.data_file_length)); + info->state= &info->s->state.state; + info->append_insert_at_end= 0; + DBUG_VOID_RETURN; +} + + +void mi_copy_status(void* to,void *from) +{ + MI_INFO *info= (MI_INFO*) to; + DBUG_ENTER("mi_copy_status"); + info->state= &((MI_INFO*) from)->save_state; + DBUG_PRINT("info",("key_file: %ld data_file: %ld", + (long) info->state->key_file_length, + (long) info->state->data_file_length)); + DBUG_VOID_RETURN; +} + + +/* + Check if should allow concurrent inserts + + IMPLEMENTATION + Allow concurrent inserts if we don't have a hole in the table or + if there is no active write lock and there is active read locks and + myisam_concurrent_insert == 2. In this last case the new + row('s) are inserted at end of file instead of filling up the hole. + + The last case is to allow one to inserts into a heavily read-used table + even if there is holes. + + NOTES + If there is a an rtree indexes in the table, concurrent inserts are + disabled in mi_open() + + RETURN + 0 ok to use concurrent inserts + 1 not ok +*/ + +my_bool mi_check_status(void *param) +{ + MI_INFO *info=(MI_INFO*) param; + DBUG_ENTER("mi_check_status"); + DBUG_PRINT("info",("dellink: %ld r_locks: %u w_locks: %u", + (long) info->s->state.dellink, (uint) info->s->r_locks, + (uint) info->s->w_locks)); + /* + The test for w_locks == 1 is here because this thread has already done an + external lock (in other words: w_locks == 1 means no other threads has + a write lock) + */ + DBUG_RETURN((my_bool) !(info->s->state.dellink == HA_OFFSET_ERROR || + (myisam_concurrent_insert == 2 && info->s->r_locks && + info->s->w_locks == 1))); +} + + +/** + Fix status for thr_lock_merge() + + @param org_table + @param new_table that should point on org_lock. new_table is 0 + in case this is the first occurrence of the table in the lock + structure. +*/ + +void mi_fix_status(MI_INFO *org_table, MI_INFO *new_table) +{ + DBUG_ENTER("mi_fix_status"); + if (!new_table) + { + /* First in group. Set state as in mi_get_status() */ + org_table->state= &org_table->save_state; + } + else + { + /* Set new_table to use state from org_table (first lock of this table) */ + new_table->state= org_table->state; + } + DBUG_VOID_RETURN; +} + + +/**************************************************************************** + ** functions to read / write the state +****************************************************************************/ + +int _mi_readinfo(register MI_INFO *info, int lock_type, int check_keybuffer) +{ + DBUG_ENTER("_mi_readinfo"); + + if (info->lock_type == F_UNLCK) + { + MYISAM_SHARE *share=info->s; + if (!share->tot_locks) + { + if (my_lock(share->kfile,lock_type,0L,F_TO_EOF, + info->lock_wait | MY_SEEK_NOT_DONE)) + DBUG_RETURN(1); + if (mi_state_info_read_dsk(share->kfile, &share->state, 1)) + { + int error= my_errno ? my_errno : HA_ERR_FILE_TOO_SHORT; + (void) my_lock(share->kfile,F_UNLCK,0L,F_TO_EOF, + MYF(MY_SEEK_NOT_DONE)); + my_errno= error; + DBUG_RETURN(1); + } + } + if (check_keybuffer) + (void) _mi_test_if_changed(info); + info->invalidator=info->s->invalidator; + } + else if (lock_type == F_WRLCK && info->lock_type == F_RDLCK) + { + my_errno=EACCES; /* Not allowed to change */ + DBUG_RETURN(-1); /* when have read_lock() */ + } + DBUG_RETURN(0); +} /* _mi_readinfo */ + + +/* + Every isam-function that updates the isam-database MUST end with this + request +*/ + +int _mi_writeinfo(register MI_INFO *info, uint operation) +{ + int error,olderror; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("_mi_writeinfo"); + DBUG_PRINT("info",("operation: %u tot_locks: %u", operation, + share->tot_locks)); + + error=0; + if (share->tot_locks == 0) + { + olderror=my_errno; /* Remember last error */ + if (operation) + { /* Two threads can't be here */ + share->state.process= share->last_process= share->this_process; + share->state.unique= info->last_unique= info->this_unique; + share->state.update_count= info->last_loop= ++info->this_loop; + if ((error=mi_state_info_write(share->kfile, &share->state, 1))) + olderror=my_errno; +#ifdef _WIN32 + if (myisam_flush) + { + if (share->file_map) + my_msync(info->dfile, share->file_map, share->mmaped_length, MS_SYNC); + mysql_file_sync(share->kfile, 0); + mysql_file_sync(info->dfile, 0); + } +#endif + } + if (!(operation & WRITEINFO_NO_UNLOCK) && + my_lock(share->kfile,F_UNLCK,0L,F_TO_EOF, + MYF(MY_WME | MY_SEEK_NOT_DONE)) && !error) + DBUG_RETURN(1); + my_errno=olderror; + } + else if (operation) + share->changed= 1; /* Mark keyfile changed */ + DBUG_RETURN(error); +} /* _mi_writeinfo */ + + + /* Test if someone has changed the database */ + /* (Should be called after readinfo) */ + +int _mi_test_if_changed(register MI_INFO *info) +{ + MYISAM_SHARE *share=info->s; + if (share->state.process != share->last_process || + share->state.unique != info->last_unique || + share->state.update_count != info->last_loop) + { /* Keyfile has changed */ + DBUG_PRINT("info",("index file changed")); + if (share->state.process != share->this_process) + (void) flush_key_blocks(share->key_cache, share->kfile, + &share->dirty_part_map, FLUSH_RELEASE); + share->last_process=share->state.process; + info->last_unique= share->state.unique; + info->last_loop= share->state.update_count; + info->update|= HA_STATE_WRITTEN; /* Must use file on next */ + info->data_changed= 1; /* For mi_is_changed */ + return 1; + } + return (!(info->update & HA_STATE_AKTIV) || + (info->update & (HA_STATE_WRITTEN | HA_STATE_DELETED | + HA_STATE_KEY_CHANGED))); +} /* _mi_test_if_changed */ + + +/* + Put a mark in the .MYI file that someone is updating the table + + + DOCUMENTATION + + state.open_count in the .MYI file is used the following way: + - For the first change of the .MYI file in this process open_count is + incremented by mi_mark_file_change(). (We have a write lock on the file + when this happens) + - In mi_close() it's decremented by _mi_decrement_open_count() if it + was incremented in the same process. + + This mean that if we are the only process using the file, the open_count + tells us if the MYISAM file wasn't properly closed. (This is true if + my_disable_locking is set). +*/ + + +int _mi_mark_file_changed(MI_INFO *info) +{ + uchar buff[3]; + register MYISAM_SHARE *share=info->s; + uint32 state; + DBUG_ENTER("_mi_mark_file_changed"); + + state= share->state.changed; + share->state.changed|= (STATE_CHANGED | STATE_NOT_ANALYZED | + STATE_NOT_OPTIMIZED_KEYS); + + if (!(state & STATE_CHANGED) || ! share->global_changed) + { + if (!share->global_changed) + { + share->global_changed=1; + share->state.open_count++; + } + if (!share->temporary) + { + mi_int2store(buff,share->state.open_count); + buff[2]=1; /* Mark that it's changed */ + DBUG_RETURN((int)mysql_file_pwrite(share->kfile, buff, sizeof(buff), + sizeof(share->state.header), + MYF(MY_NABP))); + } + } + DBUG_RETURN(0); +} + + +/* + This is only called by close or by extra(HA_FLUSH) if the OS has the pwrite() + call. In these context the following code should be safe! + */ + +int _mi_decrement_open_count(MI_INFO *info) +{ + uchar buff[2]; + register MYISAM_SHARE *share=info->s; + int lock_error=0,write_error=0; + if (share->global_changed) + { + uint old_lock=info->lock_type; + share->global_changed=0; + lock_error= my_disable_locking ? 0 : mi_lock_database(info,F_WRLCK); + /* Its not fatal even if we couldn't get the lock ! */ + if (share->state.open_count > 0) + { + share->state.open_count--; + mi_int2store(buff,share->state.open_count); + write_error= (mysql_file_pwrite(share->kfile, buff, sizeof(buff), + sizeof(share->state.header), + MYF(MY_NABP)) != 0); + } + if (!lock_error && !my_disable_locking) + lock_error=mi_lock_database(info,old_lock); + } + return MY_TEST(lock_error || write_error); +} + + +void _mi_report_crashed_ignore(MI_INFO *file __attribute__((unused)), + const char *message __attribute__((unused)), + const char *sfile __attribute__((unused)), + uint sline __attribute__((unused))) +{ +} diff --git a/storage/myisam/mi_log.c b/storage/myisam/mi_log.c new file mode 100644 index 00000000..bc9607fb --- /dev/null +++ b/storage/myisam/mi_log.c @@ -0,0 +1,157 @@ +/* Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* + Logging of MyISAM commands and records on logfile for debugging + The log can be examined with help of the myisamlog command. +*/ + +#include "myisamdef.h" +#ifdef _WIN32 +#include <fcntl.h> +#endif + +#undef GETPID /* For HPUX */ +#define GETPID() (log_type == 1 ? (long) myisam_pid : (long) my_thread_dbug_id()) + + /* Activate logging if flag is 1 and reset logging if flag is 0 */ + +static int log_type=0; +ulong myisam_pid=0; + +int mi_log(int activate_log) +{ + int error=0; + char buff[FN_REFLEN]; + DBUG_ENTER("mi_log"); + + log_type=activate_log; + if (activate_log) + { + if (!myisam_pid) + myisam_pid=(ulong) getpid(); + if (myisam_log_file < 0) + { + if ((myisam_log_file= mysql_file_create(mi_key_file_log, + fn_format(buff, + myisam_log_filename, + "", ".log", 4), + 0, + (O_RDWR | O_BINARY | O_APPEND), + MYF(0))) < 0) + DBUG_RETURN(my_errno); + } + } + else if (myisam_log_file >= 0) + { + error= mysql_file_close(myisam_log_file, MYF(0)) ? my_errno : 0 ; + myisam_log_file= -1; + } + DBUG_RETURN(error); +} + + + /* Logging of records and commands on logfile */ + /* All logs starts with command(1) dfile(2) process(4) result(2) */ + +void _myisam_log(enum myisam_log_commands command, MI_INFO *info, + const uchar *buffert, uint length) +{ + uchar buff[11]; + int error,old_errno; + ulong pid=(ulong) GETPID(); + old_errno=my_errno; + bzero(buff,sizeof(buff)); + buff[0]=(char) command; + mi_int2store(buff+1,info->dfile); + mi_int4store(buff+3,pid); + mi_int2store(buff+9,length); + + mysql_mutex_lock(&THR_LOCK_myisam); + error=my_lock(myisam_log_file,F_WRLCK,0L,F_TO_EOF,MYF(MY_SEEK_NOT_DONE)); + (void) mysql_file_write(myisam_log_file, buff, sizeof(buff), MYF(0)); + (void) mysql_file_write(myisam_log_file, buffert, length, MYF(0)); + if (!error) + error=my_lock(myisam_log_file,F_UNLCK,0L,F_TO_EOF,MYF(MY_SEEK_NOT_DONE)); + mysql_mutex_unlock(&THR_LOCK_myisam); + my_errno=old_errno; +} + + +void _myisam_log_command(enum myisam_log_commands command, MI_INFO *info, + const uchar *buffert, uint length, int result) +{ + uchar buff[9]; + int error,old_errno; + ulong pid=(ulong) GETPID(); + + old_errno=my_errno; + buff[0]=(char) command; + mi_int2store(buff+1,info->dfile); + mi_int4store(buff+3,pid); + mi_int2store(buff+7,result); + mysql_mutex_lock(&THR_LOCK_myisam); + error=my_lock(myisam_log_file,F_WRLCK,0L,F_TO_EOF,MYF(MY_SEEK_NOT_DONE)); + (void) mysql_file_write(myisam_log_file, buff, sizeof(buff), MYF(0)); + if (buffert) + (void) mysql_file_write(myisam_log_file, buffert, length, MYF(0)); + if (!error) + error=my_lock(myisam_log_file,F_UNLCK,0L,F_TO_EOF,MYF(MY_SEEK_NOT_DONE)); + mysql_mutex_unlock(&THR_LOCK_myisam); + my_errno=old_errno; +} + + +void _myisam_log_record(enum myisam_log_commands command, MI_INFO *info, + const uchar *record, my_off_t filepos, int result) +{ + uchar buff[21],*pos; + int error,old_errno; + uint length; + ulong pid=(ulong) GETPID(); + + old_errno=my_errno; + if (!info->s->base.blobs) + length=info->s->base.reclength; + else + length=info->s->base.reclength+ _mi_calc_total_blob_length(info,record); + buff[0]=(uchar) command; + mi_int2store(buff+1,info->dfile); + mi_int4store(buff+3,pid); + mi_int2store(buff+7,result); + mi_sizestore(buff+9,filepos); + mi_int4store(buff+17,length); + mysql_mutex_lock(&THR_LOCK_myisam); + error=my_lock(myisam_log_file,F_WRLCK,0L,F_TO_EOF,MYF(MY_SEEK_NOT_DONE)); + (void) mysql_file_write(myisam_log_file, buff, sizeof(buff), MYF(0)); + (void) mysql_file_write(myisam_log_file, record, info->s->base.reclength, MYF(0)); + if (info->s->base.blobs) + { + MI_BLOB *blob,*end; + + for (end=info->blobs+info->s->base.blobs, blob= info->blobs; + blob != end ; + blob++) + { + memcpy(&pos, record+blob->offset+blob->pack_length, + sizeof(char*)); + (void) mysql_file_write(myisam_log_file, pos, blob->length, MYF(0)); + } + } + if (!error) + error=my_lock(myisam_log_file,F_UNLCK,0L,F_TO_EOF,MYF(MY_SEEK_NOT_DONE)); + mysql_mutex_unlock(&THR_LOCK_myisam); + my_errno=old_errno; +} diff --git a/storage/myisam/mi_open.c b/storage/myisam/mi_open.c new file mode 100644 index 00000000..8b82a71f --- /dev/null +++ b/storage/myisam/mi_open.c @@ -0,0 +1,1388 @@ +/* + Copyright (c) 2000, 2011, Oracle and/or its affiliates + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* + open a isam-database + + Internal temporary tables + ------------------------- + Since only single instance of internal temporary table is required by + optimizer, such tables are not registered on myisam_open_list. In effect + it means (a) THR_LOCK_myisam is not held while such table is being created, + opened or closed; (b) no iteration through myisam_open_list while opening a + table. This optimization gives nice scalability benefit in concurrent + environment. MEMORY internal temporary tables are optimized similarly. +*/ + +#include "fulltext.h" +#include "sp_defs.h" +#include "rt_index.h" +#include <m_ctype.h> +#include <mysql_version.h> + +#ifdef _WIN32 +#include <fcntl.h> +#endif + +static void setup_key_functions(MI_KEYDEF *keyinfo); +#define get_next_element(to,pos,size) { memcpy((char*) to,pos,(size_t) size); \ + pos+=size;} + + +#define disk_pos_assert(pos, end_pos) \ +if (pos > end_pos) \ +{ \ + my_errno=HA_ERR_CRASHED; \ + goto err; \ +} + + +/****************************************************************************** +** Return the shared struct if the table is already open. +** In MySQL the server will handle version issues. +******************************************************************************/ + +MI_INFO *test_if_reopen(char *filename) +{ + LIST *pos; + + for (pos=myisam_open_list ; pos ; pos=pos->next) + { + MI_INFO *info=(MI_INFO*) pos->data; + MYISAM_SHARE *share=info->s; + DBUG_ASSERT(strcmp(share->unique_file_name,filename) || + share->last_version); + if (!strcmp(share->unique_file_name,filename) && share->last_version) + return info; + } + return 0; +} + + +/****************************************************************************** + open a MyISAM database. + See my_base.h for the handle_locking argument + if handle_locking and HA_OPEN_ABORT_IF_CRASHED then abort if the table + is marked crashed or if we are not using locking and the table doesn't + have an open count of 0. +******************************************************************************/ + +MI_INFO *mi_open(const char *name, int mode, uint open_flags) +{ + int lock_error,kfile,open_mode,save_errno,have_rtree=0, realpath_err; + uint i,j,len,errpos,head_length,base_pos,offset,info_length,keys, + key_parts,unique_key_parts,base_key_parts,fulltext_keys,uniques; + uint internal_table= open_flags & HA_OPEN_INTERNAL_TABLE; + char name_buff[FN_REFLEN], org_name[FN_REFLEN], index_name[FN_REFLEN], + data_name[FN_REFLEN]; + uchar *UNINIT_VAR(disk_cache), *disk_pos, *end_pos; + MI_INFO info,*UNINIT_VAR(m_info),*old_info= NULL; + MYISAM_SHARE share_buff,*share; + ulong *rec_per_key_part= 0; + my_off_t *key_root, *key_del; + ulonglong max_key_file_length, max_data_file_length; + DBUG_ENTER("mi_open"); + + kfile= -1; + lock_error=1; + errpos=0; + head_length=sizeof(share_buff.state.header); + bzero((uchar*) &info,sizeof(info)); + + realpath_err= my_realpath(name_buff, + fn_format(org_name,name,"",MI_NAME_IEXT,4),MYF(0)); + if (realpath_err > 0) /* File not found, no point in looking further. */ + { + DBUG_RETURN(NULL); + } + + if (my_is_symlink(org_name) && + (realpath_err || mysys_test_invalid_symlink(name_buff))) + { + my_errno= HA_WRONG_CREATE_OPTION; + DBUG_RETURN (NULL); + } + + if (!internal_table) + { + mysql_mutex_lock(&THR_LOCK_myisam); + old_info= test_if_reopen(name_buff); + } + + if (!old_info) + { + share= &share_buff; + bzero((uchar*) &share_buff,sizeof(share_buff)); + share_buff.key_cache= multi_key_cache_search((uchar*) name_buff, + (uint)strlen(name_buff), + dflt_key_cache); + + DBUG_EXECUTE_IF("myisam_pretend_crashed_table_on_open", + if (strstr(name, "crashed")) + { + my_errno= HA_ERR_CRASHED; + goto err; + }); + + DEBUG_SYNC_C("mi_open_kfile"); + if ((kfile= mysql_file_open(mi_key_file_kfile, name_buff, + (open_mode= O_RDWR) | O_SHARE | O_NOFOLLOW | O_CLOEXEC, + MYF(MY_NOSYMLINKS))) < 0) + { + if ((errno != EROFS && errno != EACCES) || + mode != O_RDONLY || + (kfile= mysql_file_open(mi_key_file_kfile, name_buff, + (open_mode= O_RDONLY) | O_SHARE| O_NOFOLLOW | O_CLOEXEC, + MYF(MY_NOSYMLINKS))) < 0) + goto err; + } + share->mode=open_mode; + errpos=1; + if (mysql_file_read(kfile, (uchar*)&share->state.header, head_length, + MYF(MY_NABP))) + { + my_errno= HA_ERR_NOT_A_TABLE; + goto err; + } + if (bcmp(share->state.header.file_version, myisam_file_magic, 4)) + { + DBUG_PRINT("error",("Wrong header in %s",name_buff)); + DBUG_DUMP("error_dump", share->state.header.file_version, + (size_t) head_length); + my_errno=HA_ERR_NOT_A_TABLE; + goto err; + } + share->options= mi_uint2korr(share->state.header.options); + if (share->options & + ~(HA_OPTION_PACK_RECORD | HA_OPTION_PACK_KEYS | + HA_OPTION_COMPRESS_RECORD | HA_OPTION_READ_ONLY_DATA | + HA_OPTION_TEMP_COMPRESS_RECORD | HA_OPTION_CHECKSUM | + HA_OPTION_TMP_TABLE | HA_OPTION_DELAY_KEY_WRITE | + HA_OPTION_RELIES_ON_SQL_LAYER | HA_OPTION_NULL_FIELDS)) + { + DBUG_PRINT("error",("wrong options: 0x%lx", share->options)); + my_errno=HA_ERR_OLD_FILE; + goto err; + } + if ((share->options & HA_OPTION_RELIES_ON_SQL_LAYER) && + ! (open_flags & HA_OPEN_FROM_SQL_LAYER)) + { + DBUG_PRINT("error", ("table cannot be opened from non-sql layer")); + my_errno= HA_ERR_UNSUPPORTED; + goto err; + } + /* Don't call realpath() if the name can't be a link */ + if (!strcmp(name_buff, org_name) || + my_readlink(index_name, org_name, MYF(0)) == -1) + (void) strmov(index_name, org_name); + *strrchr(org_name, '.')= '\0'; + (void) fn_format(data_name,org_name,"",MI_NAME_DEXT, + MY_APPEND_EXT|MY_UNPACK_FILENAME); + if (my_is_symlink(data_name)) + { + if (my_realpath(data_name, data_name, MYF(0))) + goto err; + if (mysys_test_invalid_symlink(data_name)) + { + my_errno= HA_WRONG_CREATE_OPTION; + goto err; + } + share->mode|= O_NOFOLLOW; /* all symlinks are resolved by realpath() */ + } + + info_length=mi_uint2korr(share->state.header.header_length); + base_pos=mi_uint2korr(share->state.header.base_pos); + if (!(disk_cache= (uchar*) my_alloca(info_length+128))) + { + my_errno=ENOMEM; + goto err; + } + end_pos=disk_cache+info_length; + errpos=2; + + mysql_file_seek(kfile, 0L, MY_SEEK_SET, MYF(0)); + if (!(open_flags & HA_OPEN_TMP_TABLE)) + { + if ((lock_error=my_lock(kfile,F_RDLCK,0L,F_TO_EOF, + MYF(open_flags & HA_OPEN_WAIT_IF_LOCKED ? + 0 : MY_SHORT_WAIT))) && + !(open_flags & HA_OPEN_IGNORE_IF_LOCKED)) + goto err; + } + errpos=3; + if (mysql_file_read(kfile, disk_cache, info_length, MYF(MY_NABP))) + { + my_errno=HA_ERR_CRASHED; + goto err; + } + len=mi_uint2korr(share->state.header.state_info_length); + keys= (uint) share->state.header.keys; + uniques= (uint) share->state.header.uniques; + fulltext_keys= (uint) share->state.header.fulltext_keys; + base_key_parts= key_parts= mi_uint2korr(share->state.header.key_parts); + unique_key_parts= mi_uint2korr(share->state.header.unique_key_parts); + if (len != MI_STATE_INFO_SIZE) + { + DBUG_PRINT("warning", + ("saved_state_info_length: %d state_info_length: %d", + len,MI_STATE_INFO_SIZE)); + } + share->state_diff_length=len-MI_STATE_INFO_SIZE; + + if (!mi_state_info_read(disk_cache, &share->state)) + goto err; + rec_per_key_part= share->state.rec_per_key_part; + key_root= share->state.key_root; + key_del= share->state.key_del; + + len= mi_uint2korr(share->state.header.base_info_length); + if (len != MI_BASE_INFO_SIZE) + { + DBUG_PRINT("warning",("saved_base_info_length: %d base_info_length: %d", + len,MI_BASE_INFO_SIZE)); + } + disk_pos= mi_n_base_info_read(disk_cache + base_pos, &share->base); + share->state.state_length=base_pos; + + if (!(open_flags & HA_OPEN_FOR_REPAIR) && + ((share->state.changed & STATE_CRASHED) || + ((open_flags & HA_OPEN_ABORT_IF_CRASHED) && + (my_disable_locking && share->state.open_count)))) + { + DBUG_PRINT("error",("Table is marked as crashed. open_flags: %u " + "changed: %u open_count: %u !locking: %d", + open_flags, share->state.changed, + share->state.open_count, my_disable_locking)); + my_errno=((share->state.changed & STATE_CRASHED_ON_REPAIR) ? + HA_ERR_CRASHED_ON_REPAIR : HA_ERR_CRASHED_ON_USAGE); + goto err; + } + + /* sanity check */ + if (share->base.keystart > 65535 || + share->base.rec_reflength > 8 || share->base.key_reflength > 7) + { + my_errno=HA_ERR_CRASHED; + goto err; + } + + key_parts+=fulltext_keys*FT_SEGS; + if (share->base.max_key_length > HA_MAX_KEY_BUFF || keys > MI_MAX_KEY || + key_parts > MI_MAX_KEY * HA_MAX_KEY_SEG) + { + DBUG_PRINT("error",("Wrong key info: Max_key_length: %d keys: %d key_parts: %d", share->base.max_key_length, keys, key_parts)); + my_errno=HA_ERR_UNSUPPORTED; + goto err; + } + + /* Correct max_file_length based on length of sizeof(off_t) */ + max_data_file_length= + (share->options & (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ? + (((ulonglong) 1 << (share->base.rec_reflength*8))-1) : + (mi_safe_mul(share->base.pack_reclength, + (ulonglong) 1 << (share->base.rec_reflength*8))-1); + max_key_file_length= + mi_safe_mul(MI_MIN_KEY_BLOCK_LENGTH, + ((ulonglong) 1 << (share->base.key_reflength*8))-1); +#if SIZEOF_OFF_T == 4 + set_if_smaller(max_data_file_length, INT_MAX32); + set_if_smaller(max_key_file_length, INT_MAX32); +#endif + share->base.max_data_file_length=(my_off_t) max_data_file_length; + share->base.max_key_file_length=(my_off_t) max_key_file_length; + + if (share->options & HA_OPTION_COMPRESS_RECORD) + share->base.max_key_length+=2; /* For safety */ + + /* Add space for node pointer */ + share->base.max_key_length+= share->base.key_reflength; + + if (!my_multi_malloc(mi_key_memory_MYISAM_SHARE, MYF(MY_WME), + &share,sizeof(*share), + &share->state.rec_per_key_part, + sizeof(long)*base_key_parts, + &share->keyinfo,keys*sizeof(MI_KEYDEF), + &share->uniqueinfo,uniques*sizeof(MI_UNIQUEDEF), + &share->keyparts, + (key_parts+unique_key_parts+keys+uniques) * + sizeof(HA_KEYSEG), + &share->rec, + (share->base.fields+1)*sizeof(MI_COLUMNDEF), + &share->blobs,sizeof(MI_BLOB)*share->base.blobs, + &share->unique_file_name,strlen(name_buff)+1, + &share->index_file_name,strlen(index_name)+1, + &share->data_file_name,strlen(data_name)+1, + &share->state.key_root,keys*sizeof(my_off_t), + &share->state.key_del, + (share->state.header.max_block_size_index*sizeof(my_off_t)), + &share->key_root_lock, sizeof(mysql_rwlock_t)*keys, + &share->mmap_lock, sizeof(mysql_rwlock_t), + NullS)) + goto err; + errpos=4; + *share=share_buff; + memcpy((char*) share->state.rec_per_key_part, + (char*) rec_per_key_part, sizeof(long)*base_key_parts); + memcpy((char*) share->state.key_root, + (char*) key_root, sizeof(my_off_t)*keys); + memcpy((char*) share->state.key_del, + (char*) key_del, (sizeof(my_off_t) * + share->state.header.max_block_size_index)); + strmov(share->unique_file_name, name_buff); + share->unique_name_length= (uint)strlen(name_buff); + strmov(share->index_file_name, index_name); + strmov(share->data_file_name, data_name); + + share->vreclength= share->base.reclength; + share->blocksize=MY_MIN(IO_SIZE,myisam_block_size); + { + HA_KEYSEG *pos=share->keyparts; + uint32 ftkey_nr= 1; + for (i=0 ; i < keys ; i++) + { + MI_KEYDEF *keyinfo= share->keyinfo + i; + keyinfo->share= share; + disk_pos=mi_keydef_read(disk_pos, keyinfo); + disk_pos_assert(disk_pos + keyinfo->keysegs * HA_KEYSEG_SIZE, end_pos); + if (keyinfo->key_alg == HA_KEY_ALG_RTREE) + have_rtree=1; + set_if_smaller(share->blocksize, keyinfo->block_length); + keyinfo->seg= pos; + for (j=0 ; j < keyinfo->keysegs; j++,pos++) + { + disk_pos=mi_keyseg_read(disk_pos, pos); + if (pos->flag & HA_BLOB_PART && + ! (share->options & (HA_OPTION_COMPRESS_RECORD | + HA_OPTION_PACK_RECORD))) + { + my_errno= HA_ERR_CRASHED; + goto err; + } + if (pos->type == HA_KEYTYPE_TEXT || + pos->type == HA_KEYTYPE_VARTEXT1 || + pos->type == HA_KEYTYPE_VARTEXT2) + { + if (!pos->language) + pos->charset=default_charset_info; + else if (!(pos->charset= get_charset(pos->language, MYF(MY_WME)))) + { + my_errno=HA_ERR_UNKNOWN_CHARSET; + goto err; + } + } + else if (pos->type == HA_KEYTYPE_BINARY) + pos->charset= &my_charset_bin; + } + if (keyinfo->flag & HA_SPATIAL) + { +#ifdef HAVE_SPATIAL + uint sp_segs= SPDIMS*2; + keyinfo->seg= pos - sp_segs; + DBUG_ASSERT(keyinfo->keysegs == sp_segs + 1); + keyinfo->keysegs= sp_segs; +#else + my_errno=HA_ERR_UNSUPPORTED; + goto err; +#endif + } + else if (keyinfo->flag & HA_FULLTEXT) + { + if (!fulltext_keys) + { /* 4.0 compatibility code, to be removed in 5.0 */ + keyinfo->seg= pos - FT_SEGS; + keyinfo->keysegs-= FT_SEGS; + } + else + { + uint k; + keyinfo->seg= pos; + for (k=0; k < FT_SEGS; k++) + { + *pos= ft_keysegs[k]; + pos[0].language= pos[-1].language; + if (!(pos[0].charset= pos[-1].charset)) + { + my_errno=HA_ERR_CRASHED; + goto err; + } + pos++; + } + } + if (!share->ft2_keyinfo.seg) + { + memcpy(& share->ft2_keyinfo, keyinfo, sizeof(MI_KEYDEF)); + share->ft2_keyinfo.keysegs=1; + share->ft2_keyinfo.flag=0; + share->ft2_keyinfo.keylength= + share->ft2_keyinfo.minlength= + share->ft2_keyinfo.maxlength=HA_FT_WLEN+share->base.rec_reflength; + share->ft2_keyinfo.seg=pos-1; + share->ft2_keyinfo.end=pos; + setup_key_functions(& share->ft2_keyinfo); + } + keyinfo->ftkey_nr= ftkey_nr++; + } + setup_key_functions(keyinfo); + keyinfo->end= pos; + pos->type=HA_KEYTYPE_END; /* End */ + pos->length=share->base.rec_reflength; + pos->null_bit=0; + pos->flag=0; /* For purify */ + pos++; + } + + for (i=0 ; i < uniques ; i++) + { + disk_pos=mi_uniquedef_read(disk_pos, &share->uniqueinfo[i]); + disk_pos_assert(disk_pos + share->uniqueinfo[i].keysegs * + HA_KEYSEG_SIZE, end_pos); + share->uniqueinfo[i].seg=pos; + for (j=0 ; j < share->uniqueinfo[i].keysegs; j++,pos++) + { + disk_pos=mi_keyseg_read(disk_pos, pos); + if (pos->type == HA_KEYTYPE_TEXT || + pos->type == HA_KEYTYPE_VARTEXT1 || + pos->type == HA_KEYTYPE_VARTEXT2) + { + if (!pos->language) + pos->charset=default_charset_info; + else if (!(pos->charset= get_charset(pos->language, MYF(MY_WME)))) + { + my_errno=HA_ERR_UNKNOWN_CHARSET; + goto err; + } + } + } + share->uniqueinfo[i].end=pos; + pos->type=HA_KEYTYPE_END; /* End */ + pos->null_bit=0; + pos->flag=0; + pos++; + } + share->ftkeys= ftkey_nr; + } + + disk_pos_assert(disk_pos + share->base.fields *MI_COLUMNDEF_SIZE, end_pos); + for (i=j=offset=0 ; i < share->base.fields ; i++) + { + disk_pos=mi_recinfo_read(disk_pos,&share->rec[i]); + share->rec[i].pack_type=0; + share->rec[i].huff_tree=0; + share->rec[i].offset=offset; + if (share->rec[i].type == FIELD_BLOB) + { + share->blobs[j].pack_length= + share->rec[i].length - portable_sizeof_char_ptr; + share->blobs[j].offset=offset; + j++; + } + /* This is to detect how to calculate checksums */ + if (share->rec[i].null_bit) + share->has_null_fields= 1; + if (share->rec[i].type == FIELD_VARCHAR) + share->has_varchar_fields= 1; + offset+=share->rec[i].length; + } + share->rec[i].type=(int) FIELD_LAST; /* End marker */ + if (offset > share->base.reclength) + { + /* purecov: begin inspected */ + my_errno= HA_ERR_CRASHED; + goto err; + /* purecov: end */ + } + + if (! lock_error) + { + (void) my_lock(kfile,F_UNLCK,0L,F_TO_EOF,MYF(MY_SEEK_NOT_DONE)); + lock_error=1; /* Database unlocked */ + } + + if (mi_open_datafile(&info, share)) + goto err; + errpos=5; + + share->kfile=kfile; + share->this_process=(ulong) getpid(); + share->last_process= share->state.process; + share->base.base_key_parts= base_key_parts; + share->base.key_parts=key_parts; + share->base.all_key_parts=key_parts+unique_key_parts; + if (!(share->last_version=share->state.version)) + share->last_version=1; /* Safety */ + share->rec_reflength=share->base.rec_reflength; /* May be changed */ + share->base.margin_key_file_length=(share->base.max_key_file_length - + (keys ? MI_INDEX_BLOCK_MARGIN * + share->blocksize * keys : 0)); + share->blocksize=MY_MIN(IO_SIZE,myisam_block_size); + share->data_file_type=STATIC_RECORD; + if (share->options & HA_OPTION_COMPRESS_RECORD) + { + share->data_file_type = COMPRESSED_RECORD; + share->options|= HA_OPTION_READ_ONLY_DATA; + info.s=share; + if (_mi_read_pack_info(&info, + (pbool) + MY_TEST(!(share->options & + (HA_OPTION_PACK_RECORD | + HA_OPTION_TEMP_COMPRESS_RECORD))))) + goto err; + } + else if (share->options & HA_OPTION_PACK_RECORD) + share->data_file_type = DYNAMIC_RECORD; + my_afree(disk_cache); + mi_setup_functions(share); + share->is_log_table= FALSE; + thr_lock_init(&share->lock); + mysql_mutex_init(mi_key_mutex_MYISAM_SHARE_intern_lock, + &share->intern_lock, MY_MUTEX_INIT_FAST); + for (i=0; i<keys; i++) + mysql_rwlock_init(mi_key_rwlock_MYISAM_SHARE_key_root_lock, + &share->key_root_lock[i]); + mysql_rwlock_init(mi_key_rwlock_MYISAM_SHARE_mmap_lock, &share->mmap_lock); + if (!thr_lock_inited) + { + /* Probably a single threaded program; Don't use concurrent inserts */ + myisam_concurrent_insert=0; + } + else if (myisam_concurrent_insert) + { + share->concurrent_insert= + ((share->options & (HA_OPTION_READ_ONLY_DATA | HA_OPTION_TMP_TABLE | + HA_OPTION_COMPRESS_RECORD | + HA_OPTION_TEMP_COMPRESS_RECORD)) || + (open_flags & HA_OPEN_TMP_TABLE) || + have_rtree) ? 0 : 1; + if (share->concurrent_insert) + { + share->lock.get_status=mi_get_status; + share->lock.copy_status=mi_copy_status; + share->lock.update_status=mi_update_status; + share->lock.restore_status= mi_restore_status; + share->lock.check_status=mi_check_status; + share->lock.fix_status= (void (*)(void *, void *)) mi_fix_status; + } + } + /* + Memory mapping can only be requested after initializing intern_lock. + */ + if (open_flags & HA_OPEN_MMAP) + { + info.s= share; + mi_extra(&info, HA_EXTRA_MMAP, 0); + } + } + else + { + share= old_info->s; + if (mode == O_RDWR && share->mode == O_RDONLY) + { + my_errno=EACCES; /* Can't open in write mode */ + goto err; + } + if (mi_open_datafile(&info, share)) + goto err; + errpos=5; + have_rtree= old_info->rtree_recursion_state != NULL; + } + + /* alloc and set up private structure parts */ + if (!my_multi_malloc(mi_key_memory_MI_INFO, MYF(MY_WME), + &m_info,sizeof(MI_INFO), + &info.blobs,sizeof(MI_BLOB)*share->base.blobs, + &info.buff,(share->base.max_key_block_length*2+ + share->base.max_key_length), + &info.lastkey,share->base.max_key_length*3+1, + &info.first_mbr_key, share->base.max_key_length, + &info.filename,strlen(name)+1, + &info.rtree_recursion_state,have_rtree ? 1024 : 0, + NullS)) + goto err; + errpos=6; + + if (!have_rtree) + info.rtree_recursion_state= NULL; + + strmov(info.filename,name); + memcpy(info.blobs,share->blobs,sizeof(MI_BLOB)*share->base.blobs); + info.lastkey2=info.lastkey+share->base.max_key_length; + + info.s=share; + info.lastpos= HA_OFFSET_ERROR; + info.update= (short) (HA_STATE_NEXT_FOUND+HA_STATE_PREV_FOUND); + info.open_flag= open_flags; + info.opt_flag=READ_CHECK_USED; + info.this_unique= (ulong) info.dfile; /* Uniq number in process */ + if (share->data_file_type == COMPRESSED_RECORD) + info.this_unique= share->state.unique; + info.this_loop=0; /* Update counter */ + info.last_unique= share->state.unique; + info.last_loop= share->state.update_count; + if (mode == O_RDONLY) + share->options|=HA_OPTION_READ_ONLY_DATA; + info.lock_type=F_UNLCK; + info.quick_mode=0; + info.bulk_insert=0; + info.ft1_to_ft2=0; + info.errkey= -1; + info.page_changed=1; + mysql_mutex_lock(&share->intern_lock); + info.read_record=share->read_record; + share->reopen++; + share->write_flag=MYF(MY_NABP | MY_WAIT_IF_FULL); + if (share->options & HA_OPTION_READ_ONLY_DATA) + { + info.lock_type=F_RDLCK; + share->r_locks++; + share->tot_locks++; + } + if ((open_flags & HA_OPEN_TMP_TABLE) || + (share->options & HA_OPTION_TMP_TABLE)) + { + share->temporary=share->delay_key_write=1; + share->write_flag=MYF(MY_NABP); + share->w_locks++; /* We don't have to update status */ + share->tot_locks++; + info.lock_type=F_WRLCK; + } + if (((open_flags & HA_OPEN_DELAY_KEY_WRITE) || + (share->options & HA_OPTION_DELAY_KEY_WRITE)) && + myisam_delay_key_write) + share->delay_key_write=1; + info.state= &share->state.state; /* Change global values by default */ + mysql_mutex_unlock(&share->intern_lock); + + /* Allocate buffer for one record */ + + /* prerequisites: bzero(info) && info->s=share; are met. */ + if (!mi_alloc_rec_buff(&info, -1, &info.rec_buff)) + goto err; + bzero(info.rec_buff, mi_get_rec_buff_len(&info, info.rec_buff)); + + *m_info=info; + thr_lock_data_init(&share->lock,&m_info->lock,(void*) m_info); + + if (!internal_table) + { + m_info->open_list.data= (void*) m_info; + myisam_open_list= list_add(myisam_open_list, &m_info->open_list); + mysql_mutex_unlock(&THR_LOCK_myisam); + } + + bzero(info.buff, share->base.max_key_block_length * 2); + my_free(rec_per_key_part); + + if (myisam_log_file >= 0) + { + intern_filename(name_buff,share->index_file_name); + _myisam_log(MI_LOG_OPEN, m_info, (uchar*) name_buff, (uint)strlen(name_buff)); + } + DBUG_RETURN(m_info); + +err: + save_errno=my_errno ? my_errno : HA_ERR_END_OF_FILE; + if ((save_errno == HA_ERR_CRASHED) || + (save_errno == HA_ERR_CRASHED_ON_USAGE) || + (save_errno == HA_ERR_CRASHED_ON_REPAIR)) + mi_report_error(save_errno, name); + switch (errpos) { + case 6: + my_free(m_info); + /* fall through */ + case 5: + (void) mysql_file_close(info.dfile, MYF(0)); + if (old_info) + break; /* Don't remove open table */ + /* fall through */ + case 4: + my_free(share); + /* fall through */ + case 3: + if (! lock_error) + (void) my_lock(kfile, F_UNLCK, 0L, F_TO_EOF, MYF(MY_SEEK_NOT_DONE)); + my_free(rec_per_key_part); + /* fall through */ + case 2: + my_afree(disk_cache); + /* fall through */ + case 1: + (void) mysql_file_close(kfile, MYF(0)); + /* fall through */ + case 0: + default: + break; + } + if (!internal_table) + mysql_mutex_unlock(&THR_LOCK_myisam); + my_errno=save_errno; + DBUG_RETURN (NULL); +} /* mi_open */ + + +uchar *mi_alloc_rec_buff(MI_INFO *info, ulong length, uchar **buf) +{ + uint extra; + uint32 UNINIT_VAR(old_length); + + if (! *buf || length > (old_length=mi_get_rec_buff_len(info, *buf))) + { + uchar *newptr = *buf; + + /* to simplify initial init of info->rec_buf in mi_open and mi_extra */ + if (length == (ulong) -1) + { + if (info->s->options & HA_OPTION_COMPRESS_RECORD) + length= MY_MAX(info->s->base.pack_reclength, info->s->max_pack_length); + else + length= info->s->base.pack_reclength; + length= MY_MAX(length, info->s->base.max_key_length); + length= MY_MAX(length, info->s->vreclength); + /* Avoid unnecessary realloc */ + if (newptr && length == old_length) + return newptr; + } + + extra= ((info->s->options & HA_OPTION_PACK_RECORD) ? + ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+ + MI_REC_BUFF_OFFSET : 0); + if (extra && newptr) + newptr-= MI_REC_BUFF_OFFSET; + if (!(newptr=(uchar*) my_realloc(mi_key_memory_record_buffer, + (uchar*)newptr, length + extra + 8, + MYF(MY_ALLOW_ZERO_PTR)))) + return NULL; + *((uint32 *) newptr)= (uint32) length; + *buf= newptr+(extra ? MI_REC_BUFF_OFFSET : 0); + } + return *buf; +} + + +ulonglong mi_safe_mul(ulonglong a, ulonglong b) +{ + ulonglong max_val= ~ (ulonglong) 0; /* my_off_t is unsigned */ + + if (!a || max_val / a < b) + return max_val; + return a*b; +} + + /* Set up functions in structs */ + +void mi_setup_functions(register MYISAM_SHARE *share) +{ + if (share->options & HA_OPTION_COMPRESS_RECORD) + { + share->read_record=_mi_read_pack_record; + share->read_rnd=_mi_read_rnd_pack_record; + if ((share->options & + (HA_OPTION_PACK_RECORD | HA_OPTION_NULL_FIELDS)) || + share->has_varchar_fields) + share->calc_checksum= mi_checksum; + else + share->calc_checksum= mi_static_checksum; + share->calc_check_checksum= share->calc_checksum; + if (!(share->options & HA_OPTION_TEMP_COMPRESS_RECORD)) + share->calc_checksum=0; /* No checksum */ + } + else if (share->options & HA_OPTION_PACK_RECORD) + { + share->read_record=_mi_read_dynamic_record; + share->read_rnd=_mi_read_rnd_dynamic_record; + share->delete_record=_mi_delete_dynamic_record; + share->compare_record=_mi_cmp_dynamic_record; + share->compare_unique=_mi_cmp_dynamic_unique; + share->calc_checksum= mi_checksum; + share->calc_check_checksum= share->calc_checksum; + + /* add bits used to pack data to pack_reclength for faster allocation */ + share->base.pack_reclength+= share->base.pack_bits; + if (share->base.blobs) + { + share->update_record=_mi_update_blob_record; + share->write_record=_mi_write_blob_record; + } + else + { + share->write_record=_mi_write_dynamic_record; + share->update_record=_mi_update_dynamic_record; + } + } + else + { + share->read_record=_mi_read_static_record; + share->read_rnd=_mi_read_rnd_static_record; + share->delete_record=_mi_delete_static_record; + share->compare_record=_mi_cmp_static_record; + share->update_record=_mi_update_static_record; + share->write_record=_mi_write_static_record; + share->compare_unique=_mi_cmp_static_unique; + if (share->options & HA_OPTION_NULL_FIELDS) + share->calc_checksum= mi_checksum; + else + share->calc_checksum= mi_static_checksum; + share->calc_check_checksum= share->calc_checksum; + } + share->file_read= mi_nommap_pread; + share->file_write= mi_nommap_pwrite; + if (!(share->options & HA_OPTION_CHECKSUM)) + share->calc_checksum=0; + return; +} + + +static void setup_key_functions(register MI_KEYDEF *keyinfo) +{ + if (keyinfo->key_alg == HA_KEY_ALG_RTREE) + { +#ifdef HAVE_RTREE_KEYS + keyinfo->ck_insert = rtree_insert; + keyinfo->ck_delete = rtree_delete; +#else + DBUG_ASSERT(0); /* mi_open should check it never happens */ +#endif + } + else + { + keyinfo->ck_insert = _mi_ck_write; + keyinfo->ck_delete = _mi_ck_delete; + } + if (keyinfo->flag & HA_BINARY_PACK_KEY) + { /* Simple prefix compression */ + keyinfo->bin_search=_mi_seq_search; + keyinfo->get_key=_mi_get_binary_pack_key; + keyinfo->pack_key=_mi_calc_bin_pack_key_length; + keyinfo->store_key=_mi_store_bin_pack_key; + } + else if (keyinfo->flag & HA_VAR_LENGTH_KEY) + { + keyinfo->get_key= _mi_get_pack_key; + if (keyinfo->seg[0].flag & HA_PACK_KEY) + { /* Prefix compression */ + /* + _mi_prefix_search() compares end-space against ASCII blank (' '). + It cannot be used for character sets, that do not encode the + blank character like ASCII does. UCS2 is an example. All + character sets with a fixed width > 1 or a mimimum width > 1 + cannot represent blank like ASCII does. In these cases we have + to use _mi_seq_search() for the search. + */ + if (!keyinfo->seg->charset || use_strnxfrm(keyinfo->seg->charset) || + (keyinfo->seg->flag & HA_NULL_PART) || + (keyinfo->seg->charset->mbminlen > 1)) + keyinfo->bin_search=_mi_seq_search; + else + keyinfo->bin_search=_mi_prefix_search; + keyinfo->pack_key=_mi_calc_var_pack_key_length; + keyinfo->store_key=_mi_store_var_pack_key; + } + else + { + keyinfo->bin_search=_mi_seq_search; + keyinfo->pack_key=_mi_calc_var_key_length; /* Variable length key */ + keyinfo->store_key=_mi_store_static_key; + } + } + else + { + keyinfo->bin_search=_mi_bin_search; + keyinfo->get_key=_mi_get_static_key; + keyinfo->pack_key=_mi_calc_static_key_length; + keyinfo->store_key=_mi_store_static_key; + } + return; +} + + +/* + Function to save and store the header in the index file (.MYI) +*/ + +uint mi_state_info_write(File file, MI_STATE_INFO *state, uint pWrite) +{ + uchar buff[MI_STATE_INFO_SIZE + MI_STATE_EXTRA_SIZE]; + uchar *ptr=buff; + uint i, keys= (uint) state->header.keys, + key_blocks=state->header.max_block_size_index; + DBUG_ENTER("mi_state_info_write"); + + memcpy(ptr, &state->header, sizeof(state->header)); + ptr+=sizeof(state->header); + + /* open_count must be first because of _mi_mark_file_changed ! */ + mi_int2store(ptr,state->open_count); ptr +=2; + *ptr++= (uchar)state->changed; *ptr++= state->sortkey; + mi_rowstore(ptr,state->state.records); ptr +=8; + mi_rowstore(ptr,state->state.del); ptr +=8; + mi_rowstore(ptr,state->split); ptr +=8; + mi_sizestore(ptr,state->dellink); ptr +=8; + mi_sizestore(ptr,state->state.key_file_length); ptr +=8; + mi_sizestore(ptr,state->state.data_file_length); ptr +=8; + mi_sizestore(ptr,state->state.empty); ptr +=8; + mi_sizestore(ptr,state->state.key_empty); ptr +=8; + mi_int8store(ptr,state->auto_increment); ptr +=8; + mi_int8store(ptr,(ulonglong) state->state.checksum);ptr +=8; + mi_int4store(ptr,state->process); ptr +=4; + mi_int4store(ptr,state->unique); ptr +=4; + mi_int4store(ptr,state->status); ptr +=4; + mi_int4store(ptr,state->update_count); ptr +=4; + + ptr+=state->state_diff_length; + + for (i=0; i < keys; i++) + { + mi_sizestore(ptr,state->key_root[i]); ptr +=8; + } + for (i=0; i < key_blocks; i++) + { + mi_sizestore(ptr,state->key_del[i]); ptr +=8; + } + if (pWrite & 2) /* From isamchk */ + { + uint key_parts= mi_uint2korr(state->header.key_parts); + mi_int4store(ptr,state->sec_index_changed); ptr +=4; + mi_int4store(ptr,state->sec_index_used); ptr +=4; + mi_int4store(ptr,state->version); ptr +=4; + mi_int8store(ptr,state->key_map); ptr +=8; + mi_int8store(ptr,(ulonglong) state->create_time); ptr +=8; + mi_int8store(ptr,(ulonglong) state->recover_time); ptr +=8; + mi_int8store(ptr,(ulonglong) state->check_time); ptr +=8; + mi_sizestore(ptr,state->rec_per_key_rows); ptr+=8; + for (i=0 ; i < key_parts ; i++) + { + mi_int4store(ptr,state->rec_per_key_part[i]); ptr+=4; + } + } + + if (pWrite & 1) + DBUG_RETURN(mysql_file_pwrite(file, buff, (size_t) (ptr-buff), 0L, + MYF(MY_NABP | MY_THREADSAFE)) != 0); + DBUG_RETURN(mysql_file_write(file, buff, (size_t) (ptr-buff), + MYF(MY_NABP)) != 0); +} + + +uchar *mi_state_info_read(uchar *ptr, MI_STATE_INFO *state) +{ + uint i,keys,key_parts,key_blocks; + memcpy(&state->header, ptr, sizeof(state->header)); + ptr +=sizeof(state->header); + keys=(uint) state->header.keys; + key_parts=mi_uint2korr(state->header.key_parts); + key_blocks=state->header.max_block_size_index; + + state->open_count = mi_uint2korr(ptr); ptr +=2; + state->changed= *ptr++; + state->sortkey = (uint) *ptr++; + state->state.records= mi_rowkorr(ptr); ptr +=8; + state->state.del = mi_rowkorr(ptr); ptr +=8; + state->split = mi_rowkorr(ptr); ptr +=8; + state->dellink= mi_sizekorr(ptr); ptr +=8; + state->state.key_file_length = mi_sizekorr(ptr); ptr +=8; + state->state.data_file_length= mi_sizekorr(ptr); ptr +=8; + state->state.empty = mi_sizekorr(ptr); ptr +=8; + state->state.key_empty= mi_sizekorr(ptr); ptr +=8; + state->auto_increment=mi_uint8korr(ptr); ptr +=8; + state->state.checksum=(ha_checksum) mi_uint8korr(ptr); ptr +=8; + state->process= mi_uint4korr(ptr); ptr +=4; + state->unique = mi_uint4korr(ptr); ptr +=4; + state->status = mi_uint4korr(ptr); ptr +=4; + state->update_count=mi_uint4korr(ptr); ptr +=4; + + ptr+= state->state_diff_length; + + if (!state->rec_per_key_part) + { + if (!my_multi_malloc(mi_key_memory_MYISAM_SHARE, MYF(MY_WME), + &state->rec_per_key_part,sizeof(long)*key_parts, + &state->key_root, keys*sizeof(my_off_t), + &state->key_del, key_blocks*sizeof(my_off_t), + NullS)) + return(0); + } + + for (i=0; i < keys; i++) + { + state->key_root[i]= mi_sizekorr(ptr); ptr +=8; + } + for (i=0; i < key_blocks; i++) + { + state->key_del[i] = mi_sizekorr(ptr); ptr +=8; + } + state->sec_index_changed = mi_uint4korr(ptr); ptr +=4; + state->sec_index_used = mi_uint4korr(ptr); ptr +=4; + state->version = mi_uint4korr(ptr); ptr +=4; + state->key_map = mi_uint8korr(ptr); ptr +=8; + state->create_time = (time_t) mi_sizekorr(ptr); ptr +=8; + state->recover_time =(time_t) mi_sizekorr(ptr); ptr +=8; + state->check_time = (time_t) mi_sizekorr(ptr); ptr +=8; + state->rec_per_key_rows=mi_sizekorr(ptr); ptr +=8; + for (i=0 ; i < key_parts ; i++) + { + state->rec_per_key_part[i]= mi_uint4korr(ptr); ptr+=4; + } + return ptr; +} + + +uint mi_state_info_read_dsk(File file, MI_STATE_INFO *state, my_bool pRead) +{ + uchar buff[MI_STATE_INFO_SIZE + MI_STATE_EXTRA_SIZE]; + + if (!myisam_single_user) + { + if (pRead) + { + if (mysql_file_pread(file, buff, state->state_length, 0L, MYF(MY_NABP))) + return 1; + } + else if (mysql_file_read(file, buff, state->state_length, MYF(MY_NABP))) + return 1; + mi_state_info_read(buff, state); + } + return 0; +} + + +/**************************************************************************** +** store and read of MI_BASE_INFO +****************************************************************************/ + +uint mi_base_info_write(File file, MI_BASE_INFO *base) +{ + uchar buff[MI_BASE_INFO_SIZE], *ptr=buff; + + mi_sizestore(ptr,base->keystart); ptr +=8; + mi_sizestore(ptr,base->max_data_file_length); ptr +=8; + mi_sizestore(ptr,base->max_key_file_length); ptr +=8; + mi_rowstore(ptr,base->records); ptr +=8; + mi_rowstore(ptr,base->reloc); ptr +=8; + mi_int4store(ptr,base->mean_row_length); ptr +=4; + mi_int4store(ptr,base->reclength); ptr +=4; + mi_int4store(ptr,base->pack_reclength); ptr +=4; + mi_int4store(ptr,base->min_pack_length); ptr +=4; + mi_int4store(ptr,base->max_pack_length); ptr +=4; + mi_int4store(ptr,base->min_block_length); ptr +=4; + mi_int4store(ptr,base->fields); ptr +=4; + mi_int4store(ptr,base->pack_fields); ptr +=4; + *ptr++=base->rec_reflength; + *ptr++=base->key_reflength; + *ptr++=base->keys; + *ptr++=base->auto_key; + mi_int2store(ptr,base->pack_bits); ptr +=2; + mi_int2store(ptr,base->blobs); ptr +=2; + mi_int2store(ptr,base->max_key_block_length); ptr +=2; + mi_int2store(ptr,base->max_key_length); ptr +=2; + mi_int2store(ptr,base->extra_alloc_bytes); ptr +=2; + *ptr++= base->extra_alloc_procent; + bzero(ptr,13); ptr +=13; /* extra */ + return mysql_file_write(file, buff, (size_t) (ptr-buff), MYF(MY_NABP)) != 0; +} + + +uchar *mi_n_base_info_read(uchar *ptr, MI_BASE_INFO *base) +{ + base->keystart = mi_sizekorr(ptr); ptr +=8; + base->max_data_file_length = mi_sizekorr(ptr); ptr +=8; + base->max_key_file_length = mi_sizekorr(ptr); ptr +=8; + base->records = (ha_rows) mi_sizekorr(ptr); ptr +=8; + base->reloc = (ha_rows) mi_sizekorr(ptr); ptr +=8; + base->mean_row_length = mi_uint4korr(ptr); ptr +=4; + base->reclength = mi_uint4korr(ptr); ptr +=4; + base->pack_reclength = mi_uint4korr(ptr); ptr +=4; + base->min_pack_length = mi_uint4korr(ptr); ptr +=4; + base->max_pack_length = mi_uint4korr(ptr); ptr +=4; + base->min_block_length = mi_uint4korr(ptr); ptr +=4; + base->fields = mi_uint4korr(ptr); ptr +=4; + base->pack_fields = mi_uint4korr(ptr); ptr +=4; + + base->rec_reflength = *ptr++; + base->key_reflength = *ptr++; + base->keys= *ptr++; + base->auto_key= *ptr++; + base->pack_bits = mi_uint2korr(ptr); ptr +=2; + base->blobs = mi_uint2korr(ptr); ptr +=2; + base->max_key_block_length= mi_uint2korr(ptr); ptr +=2; + base->max_key_length = mi_uint2korr(ptr); ptr +=2; + base->extra_alloc_bytes = mi_uint2korr(ptr); ptr +=2; + base->extra_alloc_procent = *ptr++; + + ptr+=13; + return ptr; +} + +/*-------------------------------------------------------------------------- + mi_keydef +---------------------------------------------------------------------------*/ + +uint mi_keydef_write(File file, MI_KEYDEF *keydef) +{ + uchar buff[MI_KEYDEF_SIZE]; + uchar *ptr=buff; + + *ptr++ = (uchar) keydef->keysegs; + *ptr++ = keydef->key_alg; /* Rtree or Btree */ + mi_int2store(ptr,keydef->flag); ptr +=2; + mi_int2store(ptr,keydef->block_length); ptr +=2; + mi_int2store(ptr,keydef->keylength); ptr +=2; + mi_int2store(ptr,keydef->minlength); ptr +=2; + mi_int2store(ptr,keydef->maxlength); ptr +=2; + return mysql_file_write(file, buff, (size_t) (ptr-buff), MYF(MY_NABP)) != 0; +} + +uchar *mi_keydef_read(uchar *ptr, MI_KEYDEF *keydef) +{ + keydef->keysegs = (uint) *ptr++; + keydef->key_alg = *ptr++; /* Rtree or Btree */ + + keydef->flag = mi_uint2korr(ptr); ptr +=2; + keydef->block_length = mi_uint2korr(ptr); ptr +=2; + keydef->keylength = mi_uint2korr(ptr); ptr +=2; + keydef->minlength = mi_uint2korr(ptr); ptr +=2; + keydef->maxlength = mi_uint2korr(ptr); ptr +=2; + keydef->block_size_index= keydef->block_length/MI_MIN_KEY_BLOCK_LENGTH-1; + keydef->underflow_block_length=keydef->block_length/3; + keydef->version = 0; /* Not saved */ + keydef->parser = &ft_default_parser; + keydef->ftkey_nr = 0; + return ptr; +} + +/*************************************************************************** +** mi_keyseg +***************************************************************************/ + +int mi_keyseg_write(File file, const HA_KEYSEG *keyseg) +{ + uchar buff[HA_KEYSEG_SIZE]; + uchar *ptr=buff; + ulong pos; + + *ptr++= keyseg->type; + *ptr++= keyseg->language & 0xFF; /* Collation ID, low byte */ + *ptr++= keyseg->null_bit; + *ptr++= keyseg->bit_start; + *ptr++= keyseg->language >> 8; /* Collation ID, high byte */ + *ptr++= keyseg->bit_length; + mi_int2store(ptr,keyseg->flag); ptr+=2; + mi_int2store(ptr,keyseg->length); ptr+=2; + mi_int4store(ptr,keyseg->start); ptr+=4; + pos= keyseg->null_bit ? keyseg->null_pos : keyseg->bit_pos; + mi_int4store(ptr, pos); + ptr+=4; + + return mysql_file_write(file, buff, (size_t) (ptr-buff), MYF(MY_NABP)) != 0; +} + + +uchar *mi_keyseg_read(uchar *ptr, HA_KEYSEG *keyseg) +{ + keyseg->type = *ptr++; + keyseg->language = *ptr++; + keyseg->null_bit = *ptr++; + keyseg->bit_start = *ptr++; + keyseg->language += ((uint16) (*ptr++)) << 8; + keyseg->bit_length = *ptr++; + keyseg->flag = mi_uint2korr(ptr); ptr +=2; + keyseg->length = mi_uint2korr(ptr); ptr +=2; + keyseg->start = mi_uint4korr(ptr); ptr +=4; + keyseg->null_pos = mi_uint4korr(ptr); ptr +=4; + keyseg->charset=0; /* Will be filled in later */ + if (keyseg->null_bit) + /* We adjust bit_pos if null_bit is last in the byte */ + keyseg->bit_pos= (uint16)(keyseg->null_pos + (keyseg->null_bit == (1 << 7))); + else + { + keyseg->bit_pos= (uint16)keyseg->null_pos; + keyseg->null_pos= 0; + } + return ptr; +} + +/*-------------------------------------------------------------------------- + mi_uniquedef +---------------------------------------------------------------------------*/ + +uint mi_uniquedef_write(File file, MI_UNIQUEDEF *def) +{ + uchar buff[MI_UNIQUEDEF_SIZE]; + uchar *ptr=buff; + + mi_int2store(ptr,def->keysegs); ptr+=2; + *ptr++= (uchar) def->key; + *ptr++ = (uchar) def->null_are_equal; + + return mysql_file_write(file, buff, (size_t) (ptr-buff), MYF(MY_NABP)) != 0; +} + +uchar *mi_uniquedef_read(uchar *ptr, MI_UNIQUEDEF *def) +{ + def->keysegs = mi_uint2korr(ptr); + def->key = ptr[2]; + def->null_are_equal=ptr[3]; + return ptr+4; /* 1 extra byte */ +} + +/*************************************************************************** +** MI_COLUMNDEF +***************************************************************************/ + +uint mi_recinfo_write(File file, MI_COLUMNDEF *recinfo) +{ + uchar buff[MI_COLUMNDEF_SIZE]; + uchar *ptr=buff; + + mi_int2store(ptr,recinfo->type); ptr +=2; + mi_int2store(ptr,recinfo->length); ptr +=2; + *ptr++ = recinfo->null_bit; + mi_int2store(ptr,recinfo->null_pos); ptr+= 2; + return mysql_file_write(file, buff, (size_t) (ptr-buff), MYF(MY_NABP)) != 0; +} + +uchar *mi_recinfo_read(uchar *ptr, MI_COLUMNDEF *recinfo) +{ + recinfo->type= mi_sint2korr(ptr); ptr +=2; + recinfo->length=mi_uint2korr(ptr); ptr +=2; + recinfo->null_bit= (uint8) *ptr++; + recinfo->null_pos=mi_uint2korr(ptr); ptr +=2; + return ptr; +} + +/************************************************************************** +Open data file. +We can't use dup() here as the data file descriptors need to have different +active seek-positions. +*************************************************************************/ + +int mi_open_datafile(MI_INFO *info, MYISAM_SHARE *share) +{ + myf flags= MY_WME | (share->mode & O_NOFOLLOW ? MY_NOSYMLINKS: 0); + DEBUG_SYNC_C("mi_open_datafile"); + info->dfile= mysql_file_open(mi_key_file_dfile, share->data_file_name, + share->mode | O_SHARE | O_CLOEXEC, MYF(flags)); + return info->dfile >= 0 ? 0 : 1; +} + + +int mi_open_keyfile(MYISAM_SHARE *share) +{ + if ((share->kfile= mysql_file_open(mi_key_file_kfile, + share->unique_file_name, + share->mode | O_SHARE | O_NOFOLLOW | O_CLOEXEC, + MYF(MY_NOSYMLINKS | MY_WME))) < 0) + return 1; + return 0; +} + + +/* + Disable all indexes. + + SYNOPSIS + mi_disable_indexes() + info A pointer to the MyISAM storage engine MI_INFO struct. + + DESCRIPTION + Disable all indexes. + + RETURN + 0 ok +*/ + +int mi_disable_indexes(MI_INFO *info) +{ + MYISAM_SHARE *share= info->s; + + mi_clear_all_keys_active(share->state.key_map); + return 0; +} + + +/* + Enable all indexes + + SYNOPSIS + mi_enable_indexes() + info A pointer to the MyISAM storage engine MI_INFO struct. + + DESCRIPTION + Enable all indexes. The indexes might have been disabled + by mi_disable_index() before. + The function works only if both data and indexes are empty, + otherwise a repair is required. + To be sure, call handler::delete_all_rows() before. + + RETURN + 0 ok + HA_ERR_CRASHED data or index is non-empty. +*/ + +int mi_enable_indexes(MI_INFO *info) +{ + int error= 0; + MYISAM_SHARE *share= info->s; + + if (share->state.state.data_file_length || + (share->state.state.key_file_length != share->base.keystart)) + { + mi_print_error(info->s, HA_ERR_CRASHED); + error= HA_ERR_CRASHED; + } + else + mi_set_all_keys_active(share->state.key_map, share->base.keys); + return error; +} + + +/* + Test if indexes are disabled. + + SYNOPSIS + mi_indexes_are_disabled() + info A pointer to the MyISAM storage engine MI_INFO struct. + + DESCRIPTION + Test if indexes are disabled. + + RETURN + 0 indexes are not disabled + 1 all indexes are disabled + 2 non-unique indexes are disabled +*/ + +int mi_indexes_are_disabled(MI_INFO *info) +{ + MYISAM_SHARE *share= info->s; + + /* + No keys or all are enabled. keys is the number of keys. Left shifted + gives us only one bit set. When decreased by one, gives us all all bits + up to this one set and it gets unset. + */ + if (!share->base.keys || + (mi_is_all_keys_active(share->state.key_map, share->base.keys))) + return 0; + + /* All are disabled */ + if (mi_is_any_key_active(share->state.key_map)) + return 1; + + /* + We have keys. Some enabled, some disabled. + Don't check for any non-unique disabled but return directly 2 + */ + return 2; +} diff --git a/storage/myisam/mi_packrec.c b/storage/myisam/mi_packrec.c new file mode 100644 index 00000000..ca8a8ef0 --- /dev/null +++ b/storage/myisam/mi_packrec.c @@ -0,0 +1,1714 @@ +/* Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2020, MariaDB Corporation. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + + /* Functions to compressed records */ + +#include "fulltext.h" + +#define IS_CHAR ((uint) 32768) /* Bit if char (not offset) in tree */ + +/* Some definitions to keep in sync with myisampack.c */ +#define HEAD_LENGTH 32 /* Length of fixed header */ + +#if INT_MAX > 32767 +#define BITS_SAVED 32 +#define MAX_QUICK_TABLE_BITS 9 /* Because we may shift in 24 bits */ +#else +#define BITS_SAVED 16 +#define MAX_QUICK_TABLE_BITS 6 +#endif + +#define get_bit(BU) ((BU)->bits ? \ + (BU)->current_byte & ((mi_bit_type) 1 << --(BU)->bits) :\ + (fill_buffer(BU), (BU)->bits= BITS_SAVED-1,\ + (BU)->current_byte & ((mi_bit_type) 1 << (BITS_SAVED-1)))) +#define skip_to_next_byte(BU) ((BU)->bits&=~7) +#define get_bits(BU,count) (((BU)->bits >= count) ? (((BU)->current_byte >> ((BU)->bits-=count)) & mask[count]) : fill_and_get_bits(BU,count)) + +#define decode_bytes_test_bit(bit) \ + if (low_byte & (1 << (7-bit))) \ + pos++; \ + if (*pos & IS_CHAR) \ + { bits-=(bit+1); break; } \ + pos+= *pos + +/* Size in uint16 of a Huffman tree for byte compression of 256 byte values. */ +#define OFFSET_TABLE_SIZE 512 + +static uint read_huff_table(MI_BIT_BUFF *bit_buff,MI_DECODE_TREE *decode_tree, + uint16 **decode_table,uchar **intervall_buff, + uint16 *tmp_buff); +static void make_quick_table(uint16 *to_table,uint16 *decode_table, + uint *next_free,uint value,uint bits, + uint max_bits); +static void fill_quick_table(uint16 *table,uint bits, uint max_bits, + uint value); +static uint copy_decode_table(uint16 *to_pos,uint offset, + uint16 *decode_table); +static uint find_longest_bitstream(uint16 *table, uint16 *end); +static void (*get_unpack_function(MI_COLUMNDEF *rec))(MI_COLUMNDEF *field, + MI_BIT_BUFF *buff, + uchar *to, + uchar *end); +static void uf_zerofill_skip_zero(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_skip_zero(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_space_normal(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_space_endspace_selected(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end); +static void uf_endspace_selected(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_space_endspace(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_endspace(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_space_prespace_selected(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end); +static void uf_prespace_selected(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_space_prespace(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_prespace(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_zerofill_normal(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_constant(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_intervall(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_zero(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static void uf_blob(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end); +static void uf_varchar1(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end); +static void uf_varchar2(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end); +static void decode_bytes(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff, + uchar *to,uchar *end); +static uint decode_pos(MI_BIT_BUFF *bit_buff,MI_DECODE_TREE *decode_tree); +static void init_bit_buffer(MI_BIT_BUFF *bit_buff,uchar *buffer,uint length); +static uint fill_and_get_bits(MI_BIT_BUFF *bit_buff,uint count); +static void fill_buffer(MI_BIT_BUFF *bit_buff); +static uint max_bit(uint value); +static uint read_pack_length(uint version, const uchar *buf, ulong *length); +#ifdef HAVE_MMAP +static uchar *_mi_mempack_get_block_info(MI_INFO *myisam, MI_BIT_BUFF *bit_buff, + MI_BLOCK_INFO *info, uchar **rec_buff_p, + uchar *header); +#endif + +static mi_bit_type mask[]= +{ + 0x00000000, + 0x00000001, 0x00000003, 0x00000007, 0x0000000f, + 0x0000001f, 0x0000003f, 0x0000007f, 0x000000ff, + 0x000001ff, 0x000003ff, 0x000007ff, 0x00000fff, + 0x00001fff, 0x00003fff, 0x00007fff, 0x0000ffff, +#if BITS_SAVED > 16 + 0x0001ffff, 0x0003ffff, 0x0007ffff, 0x000fffff, + 0x001fffff, 0x003fffff, 0x007fffff, 0x00ffffff, + 0x01ffffff, 0x03ffffff, 0x07ffffff, 0x0fffffff, + 0x1fffffff, 0x3fffffff, 0x7fffffff, 0xffffffff, +#endif + }; + + + /* Read all packed info, allocate memory and fix field structs */ + +my_bool _mi_read_pack_info(MI_INFO *info, pbool fix_keys) +{ + File file; + int diff_length; + uint i,trees,huff_tree_bits,rec_reflength,length; + uint16 *decode_table,*tmp_buff; + ulong elements,intervall_length; + uchar *disk_cache; + uchar *intervall_buff; + uchar header[HEAD_LENGTH]; + MYISAM_SHARE *share=info->s; + MI_BIT_BUFF bit_buff; + DBUG_ENTER("_mi_read_pack_info"); + + if (myisam_quick_table_bits < 4) + myisam_quick_table_bits=4; + else if (myisam_quick_table_bits > MAX_QUICK_TABLE_BITS) + myisam_quick_table_bits=MAX_QUICK_TABLE_BITS; + + file=info->dfile; + my_errno=0; + if (mysql_file_read(file, (uchar*) header, sizeof(header), MYF(MY_NABP))) + { + if (!my_errno) + my_errno=HA_ERR_END_OF_FILE; + goto err0; + } + /* Only the first three bytes of magic number are independent of version. */ + if (memcmp((uchar*) header, (uchar*) myisam_pack_file_magic, 3)) + { + my_errno=HA_ERR_WRONG_IN_RECORD; + goto err0; + } + share->pack.version= header[3]; /* fourth byte of magic number */ + share->pack.header_length= uint4korr(header+4); + share->min_pack_length=(uint) uint4korr(header+8); + share->max_pack_length=(uint) uint4korr(header+12); + elements=uint4korr(header+16); + intervall_length=uint4korr(header+20); + trees=uint2korr(header+24); + share->pack.ref_length=header[26]; + rec_reflength=header[27]; + diff_length=(int) rec_reflength - (int) share->base.rec_reflength; + if (fix_keys) + share->rec_reflength=rec_reflength; + share->base.min_block_length=share->min_pack_length+1; + if (share->min_pack_length > 254) + share->base.min_block_length+=2; + DBUG_PRINT("info", ("fixed header length: %u", HEAD_LENGTH)); + DBUG_PRINT("info", ("total header length: %lu", share->pack.header_length)); + DBUG_PRINT("info", ("pack file version: %u", share->pack.version)); + DBUG_PRINT("info", ("min pack length: %lu", share->min_pack_length)); + DBUG_PRINT("info", ("max pack length: %lu", share->max_pack_length)); + DBUG_PRINT("info", ("elements of all trees: %lu", elements)); + DBUG_PRINT("info", ("distinct values bytes: %lu", intervall_length)); + DBUG_PRINT("info", ("number of code trees: %u", trees)); + DBUG_PRINT("info", ("bytes for record lgt: %u", share->pack.ref_length)); + DBUG_PRINT("info", ("record pointer length: %u", rec_reflength)); + + /* + Memory segment #1: + - Decode tree heads + - Distinct column values + */ + if (!(share->decode_trees=(MI_DECODE_TREE*) + my_malloc(mi_key_memory_MI_DECODE_TREE, + trees*sizeof(MI_DECODE_TREE) + intervall_length*sizeof(uchar), + MYF(MY_WME)))) + goto err0; + intervall_buff=(uchar*) (share->decode_trees+trees); + + /* + Memory segment #2: + - Decode tables + - Quick decode tables + - Temporary decode table + - Compressed data file header cache + This segment will be reallocated after construction of the tables. + */ + length=(uint) (elements*2+trees*(1 << myisam_quick_table_bits)); + /* + To keep some algorithms simpler, we accept that they access + bytes beyond the end of the input data. This can affect up to + one byte less than the "word size" size used in this file, + which is BITS_SAVED / 8. To avoid accessing non-allocated + data, we add (BITS_SAVED / 8) - 1 bytes to the buffer size. + */ + if (!(share->decode_tables=(uint16*) + my_malloc(mi_key_memory_MYISAM_SHARE_decode_tables, + (length + OFFSET_TABLE_SIZE) * sizeof(uint16) + + (uint) (share->pack.header_length - sizeof(header) + + (BITS_SAVED / 8) - 1), MYF(MY_WME | MY_ZEROFILL)))) + goto err1; + tmp_buff=share->decode_tables+length; + disk_cache= (uchar*) (tmp_buff+OFFSET_TABLE_SIZE); + + if (mysql_file_read(file, disk_cache, + (uint) (share->pack.header_length-sizeof(header)), + MYF(MY_NABP))) + goto err2; + + huff_tree_bits=max_bit(trees ? trees-1 : 0); + init_bit_buffer(&bit_buff, disk_cache, + (uint) (share->pack.header_length-sizeof(header))); + /* Read new info for each field */ + for (i=0 ; i < share->base.fields ; i++) + { + share->rec[i].base_type=(enum en_fieldtype) get_bits(&bit_buff,5); + share->rec[i].pack_type=(uint) get_bits(&bit_buff,6); + share->rec[i].space_length_bits=get_bits(&bit_buff,5); + share->rec[i].huff_tree=share->decode_trees+(uint) get_bits(&bit_buff, + huff_tree_bits); + share->rec[i].unpack=get_unpack_function(share->rec+i); + DBUG_PRINT("info", ("col: %2u type: %2u pack: %u slbits: %2u", + i, share->rec[i].base_type, share->rec[i].pack_type, + share->rec[i].space_length_bits)); + } + skip_to_next_byte(&bit_buff); + /* + Construct the decoding tables from the file header. Keep track of + the used memory. + */ + decode_table=share->decode_tables; + for (i=0 ; i < trees ; i++) + if (read_huff_table(&bit_buff,share->decode_trees+i,&decode_table, + &intervall_buff,tmp_buff)) + goto err3; + /* Reallocate the decoding tables to the used size. */ + decode_table=(uint16*) + my_realloc(mi_key_memory_MYISAM_SHARE_decode_tables, + (uchar*) share->decode_tables, + (uint) ((uchar*) decode_table - (uchar*) share->decode_tables), + MYF(0)); + /* Fix the table addresses in the tree heads. */ + { + my_ptrdiff_t diff=PTR_BYTE_DIFF(decode_table,share->decode_tables); + share->decode_tables=decode_table; + for (i=0 ; i < trees ; i++) + share->decode_trees[i].table=ADD_TO_PTR(share->decode_trees[i].table, + diff, uint16*); + } + + /* Fix record-ref-length for keys */ + if (fix_keys) + { + for (i=0 ; i < share->base.keys ; i++) + { + MI_KEYDEF *keyinfo= &share->keyinfo[i]; + keyinfo->keylength+= (uint16) diff_length; + keyinfo->minlength+= (uint16) diff_length; + keyinfo->maxlength+= (uint16) diff_length; + keyinfo->seg[keyinfo->flag & HA_FULLTEXT ? + FT_SEGS : keyinfo->keysegs].length= (uint16) rec_reflength; + } + if (share->ft2_keyinfo.seg) + { + MI_KEYDEF *ft2_keyinfo= &share->ft2_keyinfo; + ft2_keyinfo->keylength+= (uint16) diff_length; + ft2_keyinfo->minlength+= (uint16) diff_length; + ft2_keyinfo->maxlength+= (uint16) diff_length; + } + } + + if (bit_buff.error || bit_buff.pos < bit_buff.end) + goto err3; + + DBUG_RETURN(0); + +err3: + my_errno=HA_ERR_WRONG_IN_RECORD; +err2: + my_free(share->decode_tables); +err1: + my_free(share->decode_trees); +err0: + DBUG_RETURN(1); +} + + +/* + Read a huff-code-table from datafile. + + SYNOPSIS + read_huff_table() + bit_buff Bit buffer pointing at start of the + decoding table in the file header cache. + decode_tree Pointer to the decode tree head. + decode_table IN/OUT Address of a pointer to the next free space. + intervall_buff IN/OUT Address of a pointer to the next unused values. + tmp_buff Buffer for temporary extraction of a full + decoding table as read from bit_buff. + + RETURN + 0 OK. + 1 Error. +*/ + +static uint read_huff_table(MI_BIT_BUFF *bit_buff, MI_DECODE_TREE *decode_tree, + uint16 **decode_table, uchar **intervall_buff, + uint16 *tmp_buff) +{ + uint min_chr,elements,char_bits,offset_bits,size,intervall_length,table_bits, + next_free_offset; + uint16 *ptr,*end; + DBUG_ENTER("read_huff_table"); + + if (!get_bits(bit_buff,1)) + { + /* Byte value compression. */ + min_chr=get_bits(bit_buff,8); + elements=get_bits(bit_buff,9); + char_bits=get_bits(bit_buff,5); + offset_bits=get_bits(bit_buff,5); + intervall_length=0; + ptr=tmp_buff; + DBUG_PRINT("info", ("byte value compression")); + DBUG_PRINT("info", ("minimum byte value: %u", min_chr)); + DBUG_PRINT("info", ("number of tree nodes: %u", elements)); + DBUG_PRINT("info", ("bits for values: %u", char_bits)); + DBUG_PRINT("info", ("bits for tree offsets: %u", offset_bits)); + if (elements > 256) + { + DBUG_PRINT("error", ("ERROR: illegal number of tree elements: %u", + elements)); + DBUG_RETURN(1); + } + } + else + { + /* Distinct column value compression. */ + min_chr=0; + elements=get_bits(bit_buff,15); + intervall_length=get_bits(bit_buff,16); + char_bits=get_bits(bit_buff,5); + offset_bits=get_bits(bit_buff,5); + decode_tree->quick_table_bits=0; + ptr= *decode_table; + DBUG_PRINT("info", ("distinct column value compression")); + DBUG_PRINT("info", ("number of tree nodes: %u", elements)); + DBUG_PRINT("info", ("value buffer length: %u", intervall_length)); + DBUG_PRINT("info", ("bits for value index: %u", char_bits)); + DBUG_PRINT("info", ("bits for tree offsets: %u", offset_bits)); + } + size=elements*2-2; + DBUG_PRINT("info", ("tree size in uint16: %u", size)); + DBUG_PRINT("info", ("tree size in bytes: %u", + size * (uint) sizeof(uint16))); + + for (end=ptr+size ; ptr < end ; ptr++) + { + if (get_bit(bit_buff)) + { + *ptr= (uint16) get_bits(bit_buff,offset_bits); + if ((ptr + *ptr >= end) || !*ptr) + { + DBUG_PRINT("error", ("ERROR: illegal pointer in decode tree")); + DBUG_RETURN(1); + } + } + else + *ptr= (uint16) (IS_CHAR + (get_bits(bit_buff,char_bits) + min_chr)); + } + skip_to_next_byte(bit_buff); + + decode_tree->table= *decode_table; + decode_tree->intervalls= *intervall_buff; + if (! intervall_length) + { + /* Byte value compression. ptr started from tmp_buff. */ + /* Find longest Huffman code from begin to end of tree in bits. */ + table_bits= find_longest_bitstream(tmp_buff, ptr); + if (table_bits >= OFFSET_TABLE_SIZE) + DBUG_RETURN(1); + if (table_bits > myisam_quick_table_bits) + table_bits=myisam_quick_table_bits; + DBUG_PRINT("info", ("table bits: %u", table_bits)); + + next_free_offset= (1 << table_bits); + make_quick_table(*decode_table,tmp_buff,&next_free_offset,0,table_bits, + table_bits); + (*decode_table)+= next_free_offset; + decode_tree->quick_table_bits=table_bits; + } + else + { + /* Distinct column value compression. ptr started from *decode_table */ + (*decode_table)=end; + /* + get_bits() moves some bytes to a cache buffer in advance. May need + to step back. + */ + bit_buff->pos-= bit_buff->bits/8; + /* Copy the distinct column values from the buffer. */ + memcpy(*intervall_buff,bit_buff->pos,(size_t) intervall_length); + (*intervall_buff)+=intervall_length; + bit_buff->pos+=intervall_length; + bit_buff->bits=0; + } + DBUG_RETURN(0); +} + + +/* + Make a quick_table for faster decoding. + + SYNOPSIS + make_quick_table() + to_table Target quick_table and remaining decode table. + decode_table Source Huffman (sub-)tree within tmp_buff. + next_free_offset IN/OUT Next free offset from to_table. + Starts behind quick_table on the top-level. + value Huffman bits found so far. + bits Remaining bits to be collected. + max_bits Total number of bits to collect (table_bits). + + DESCRIPTION + + The quick table is an array of 16-bit values. There exists one value + for each possible code representable by max_bits (table_bits) bits. + In most cases table_bits is 9. So there are 512 16-bit values. + + If the high-order bit (16) is set (IS_CHAR) then the array slot for + this value is a valid Huffman code for a resulting byte value. + + The low-order 8 bits (1..8) are the resulting byte value. + + Bits 9..14 are the length of the Huffman code for this byte value. + This means so many bits from the input stream were needed to + represent this byte value. The remaining bits belong to later + Huffman codes. This also means that for every Huffman code shorter + than table_bits there are multiple entires in the array, which + differ just in the unused bits. + + If the high-order bit (16) is clear (0) then the remaining bits are + the position of the remaining Huffman decode tree segment behind the + quick table. + + RETURN + void +*/ + +static void make_quick_table(uint16 *to_table, uint16 *decode_table, + uint *next_free_offset, uint value, uint bits, + uint max_bits) +{ + DBUG_ENTER("make_quick_table"); + + /* + When down the table to the requested maximum, copy the rest of the + Huffman table. + */ + if (!bits--) + { + /* + Remaining left Huffman tree segment starts behind quick table. + Remaining right Huffman tree segment starts behind left segment. + */ + to_table[value]= (uint16) *next_free_offset; + /* + Re-construct the remaining Huffman tree segment at + next_free_offset in to_table. + */ + *next_free_offset= copy_decode_table(to_table, *next_free_offset, + decode_table); + DBUG_VOID_RETURN; + } + + /* Descent on the left side. Left side bits are clear (0). */ + if (!(*decode_table & IS_CHAR)) + { + /* Not a leaf. Follow the pointer. */ + make_quick_table(to_table, decode_table + *decode_table, + next_free_offset, value, bits, max_bits); + } + else + { + /* + A leaf. A Huffman code is complete. Fill the quick_table + array for all possible bit strings starting with this Huffman + code. + */ + fill_quick_table(to_table + value, bits, max_bits, (uint) *decode_table); + } + + /* Descent on the right side. Right side bits are set (1). */ + decode_table++; + value|= (1 << bits); + if (!(*decode_table & IS_CHAR)) + { + /* Not a leaf. Follow the pointer. */ + make_quick_table(to_table, decode_table + *decode_table, + next_free_offset, value, bits, max_bits); + } + else + { + /* + A leaf. A Huffman code is complete. Fill the quick_table + array for all possible bit strings starting with this Huffman + code. + */ + fill_quick_table(to_table + value, bits, max_bits, (uint) *decode_table); + } + + DBUG_VOID_RETURN; +} + + +/* + Fill quick_table for all possible values starting with this Huffman code. + + SYNOPSIS + fill_quick_table() + table Target quick_table position. + bits Unused bits from max_bits. + max_bits Total number of bits to collect (table_bits). + value The byte encoded by the found Huffman code. + + DESCRIPTION + + Fill the segment (all slots) of the quick_table array with the + resulting value for the found Huffman code. There are as many slots + as there are combinations representable by the unused bits. + + In most cases we use 9 table bits. Assume a 3-bit Huffman code. Then + there are 6 unused bits. Hence we fill 2**6 = 64 slots with the + value. + + RETURN + void +*/ + +static void fill_quick_table(uint16 *table, uint bits, uint max_bits, + uint value) +{ + uint16 *end; + DBUG_ENTER("fill_quick_table"); + + /* + Bits 1..8 of value represent the decoded byte value. + Bits 9..14 become the length of the Huffman code for this byte value. + Bit 16 flags a valid code (IS_CHAR). + */ + value|= (max_bits - bits) << 8 | IS_CHAR; + + for (end= table + ((my_ptrdiff_t) 1 << bits); table < end; table++) + { + *table= (uint16) value; + } + DBUG_VOID_RETURN; +} + + +/* + Reconstruct a decode subtree at the target position. + + SYNOPSIS + copy_decode_table() + to_pos Target quick_table and remaining decode table. + offset Next free offset from to_pos. + decode_table Source Huffman subtree within tmp_buff. + + NOTE + Pointers in the decode tree are relative to the pointers position. + + RETURN + next free offset from to_pos. +*/ + +static uint copy_decode_table(uint16 *to_pos, uint offset, + uint16 *decode_table) +{ + uint prev_offset= offset; + DBUG_ENTER("copy_decode_table"); + + /* Descent on the left side. */ + if (!(*decode_table & IS_CHAR)) + { + /* Set a pointer to the next target node. */ + to_pos[offset]=2; + /* Copy the left hand subtree there. */ + offset=copy_decode_table(to_pos,offset+2,decode_table+ *decode_table); + } + else + { + /* Copy the byte value. */ + to_pos[offset]= *decode_table; + /* Step behind this node. */ + offset+=2; + } + + /* Descent on the right side. */ + decode_table++; + if (!(*decode_table & IS_CHAR)) + { + /* Set a pointer to the next free target node. */ + to_pos[prev_offset+1]=(uint16) (offset-prev_offset-1); + /* Copy the right hand subtree to the entry of that node. */ + offset=copy_decode_table(to_pos,offset,decode_table+ *decode_table); + } + else + { + /* Copy the byte value. */ + to_pos[prev_offset+1]= *decode_table; + } + DBUG_RETURN(offset); +} + + +/* + Find the length of the longest Huffman code in this table in bits. + + SYNOPSIS + find_longest_bitstream() + table Code (sub-)table start. + end End of code table. + + IMPLEMENTATION + + Recursively follow the branch(es) of the code pair on every level of + the tree until two byte values (and no branch) are found. Add one to + each level when returning back from each recursion stage. + + 'end' is used for error checking only. A clean tree terminates + before reaching 'end'. Hence the exact value of 'end' is not too + important. However having it higher than necessary could lead to + misbehaviour should 'next' jump into the dirty area. + + RETURN + length Length of longest Huffman code in bits. + >= OFFSET_TABLE_SIZE Error, broken tree. It does not end before 'end'. +*/ + +static uint find_longest_bitstream(uint16 *table, uint16 *end) +{ + uint length= 1; + uint length2; + + if (!(*table & IS_CHAR)) + { + uint16 *next= table + *table; + if (next > end || next == table) + { + DBUG_PRINT("error", ("ERROR: illegal pointer in decode tree")); + return OFFSET_TABLE_SIZE; + } + length= find_longest_bitstream(next, end) + 1; + } + table++; + if (!(*table & IS_CHAR)) + { + uint16 *next= table + *table; + if (next > end || next == table) + { + DBUG_PRINT("error", ("ERROR: illegal pointer in decode tree")); + return OFFSET_TABLE_SIZE; + } + length2= find_longest_bitstream(next, end) + 1; + length=MY_MAX(length,length2); + } + return length; +} + + +/* + Read record from datafile. + + SYNOPSIS + _mi_read_pack_record() + info A pointer to MI_INFO. + filepos File offset of the record. + buf RETURN The buffer to receive the record. + + RETURN + 0 on success + HA_ERR_WRONG_IN_RECORD or -1 on error +*/ + +int _mi_read_pack_record(MI_INFO *info, my_off_t filepos, uchar *buf) +{ + MI_BLOCK_INFO block_info; + File file; + DBUG_ENTER("mi_read_pack_record"); + + if (filepos == HA_OFFSET_ERROR) + DBUG_RETURN(-1); /* _search() didn't find record */ + + file=info->dfile; + if (_mi_pack_get_block_info(info, &info->bit_buff, &block_info, + &info->rec_buff, file, filepos)) + goto err; + if (mysql_file_read(file, (uchar*) info->rec_buff + block_info.offset, + block_info.rec_len - block_info.offset, MYF(MY_NABP))) + goto panic; + info->update|= HA_STATE_AKTIV; + + info->rec_buff[block_info.rec_len]= 0; /* Keep valgrind happy */ + DBUG_RETURN(_mi_pack_rec_unpack(info, &info->bit_buff, buf, + info->rec_buff, block_info.rec_len)); +panic: + my_errno=HA_ERR_WRONG_IN_RECORD; +err: + DBUG_RETURN(-1); +} + + + +int _mi_pack_rec_unpack(register MI_INFO *info, MI_BIT_BUFF *bit_buff, + register uchar *to, uchar *from, ulong reclength) +{ + uchar *end_field; + reg3 MI_COLUMNDEF *end; + MI_COLUMNDEF *current_field; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("_mi_pack_rec_unpack"); + + init_bit_buffer(bit_buff, (uchar*) from, reclength); + + for (current_field=share->rec, end=current_field+share->base.fields ; + current_field < end ; + current_field++,to=end_field) + { + end_field=to+current_field->length; + (*current_field->unpack)(current_field, bit_buff, (uchar*) to, + (uchar*) end_field); + } + if (!bit_buff->error && + bit_buff->pos - bit_buff->bits / 8 == bit_buff->end) + DBUG_RETURN(0); + info->update&= ~HA_STATE_AKTIV; + DBUG_RETURN(my_errno=HA_ERR_WRONG_IN_RECORD); +} /* _mi_pack_rec_unpack */ + + + /* Return function to unpack field */ + +static void (*get_unpack_function(MI_COLUMNDEF *rec)) +(MI_COLUMNDEF *, MI_BIT_BUFF *, uchar *, uchar *) +{ + switch (rec->base_type) { + case FIELD_SKIP_ZERO: + if (rec->pack_type & PACK_TYPE_ZERO_FILL) + return &uf_zerofill_skip_zero; + return &uf_skip_zero; + case FIELD_NORMAL: + if (rec->pack_type & PACK_TYPE_SPACE_FIELDS) + return &uf_space_normal; + if (rec->pack_type & PACK_TYPE_ZERO_FILL) + return &uf_zerofill_normal; + return &decode_bytes; + case FIELD_SKIP_ENDSPACE: + if (rec->pack_type & PACK_TYPE_SPACE_FIELDS) + { + if (rec->pack_type & PACK_TYPE_SELECTED) + return &uf_space_endspace_selected; + return &uf_space_endspace; + } + if (rec->pack_type & PACK_TYPE_SELECTED) + return &uf_endspace_selected; + return &uf_endspace; + case FIELD_SKIP_PRESPACE: + if (rec->pack_type & PACK_TYPE_SPACE_FIELDS) + { + if (rec->pack_type & PACK_TYPE_SELECTED) + return &uf_space_prespace_selected; + return &uf_space_prespace; + } + if (rec->pack_type & PACK_TYPE_SELECTED) + return &uf_prespace_selected; + return &uf_prespace; + case FIELD_CONSTANT: + return &uf_constant; + case FIELD_INTERVALL: + return &uf_intervall; + case FIELD_ZERO: + case FIELD_CHECK: + return &uf_zero; + case FIELD_BLOB: + return &uf_blob; + case FIELD_VARCHAR: + if (rec->length <= 256) /* 255 + 1 byte length */ + return &uf_varchar1; + return &uf_varchar2; + case FIELD_LAST: + default: + return 0; /* This should never happen */ + } +} + + /* The different functions to unpack a field */ + +static void uf_zerofill_skip_zero(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end) +{ + if (get_bit(bit_buff)) + bzero((char*) to,(uint) (end-to)); + else + { + end-=rec->space_length_bits; + decode_bytes(rec,bit_buff,to,end); + bzero((char*) end,rec->space_length_bits); + } +} + +static void uf_skip_zero(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, uchar *to, + uchar *end) +{ + if (get_bit(bit_buff)) + bzero((char*) to,(uint) (end-to)); + else + decode_bytes(rec,bit_buff,to,end); +} + +static void uf_space_normal(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, uchar *to, + uchar *end) +{ + if (get_bit(bit_buff)) + bfill((uchar*) to,(end-to),' '); + else + decode_bytes(rec,bit_buff,to,end); +} + +static void uf_space_endspace_selected(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end) +{ + uint spaces; + if (get_bit(bit_buff)) + bfill((uchar*) to,(end-to),' '); + else + { + if (get_bit(bit_buff)) + { + if ((spaces=get_bits(bit_buff,rec->space_length_bits))+to > end) + { + bit_buff->error=1; + return; + } + if (to+spaces != end) + decode_bytes(rec,bit_buff,to,end-spaces); + bfill((uchar*) end-spaces,spaces,' '); + } + else + decode_bytes(rec,bit_buff,to,end); + } +} + +static void uf_endspace_selected(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end) +{ + uint spaces; + if (get_bit(bit_buff)) + { + if ((spaces=get_bits(bit_buff,rec->space_length_bits))+to > end) + { + bit_buff->error=1; + return; + } + if (to+spaces != end) + decode_bytes(rec,bit_buff,to,end-spaces); + bfill((uchar*) end-spaces,spaces,' '); + } + else + decode_bytes(rec,bit_buff,to,end); +} + +static void uf_space_endspace(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, uchar *to, + uchar *end) +{ + uint spaces; + if (get_bit(bit_buff)) + bfill((uchar*) to,(end-to),' '); + else + { + if ((spaces=get_bits(bit_buff,rec->space_length_bits))+to > end) + { + bit_buff->error=1; + return; + } + if (to+spaces != end) + decode_bytes(rec,bit_buff,to,end-spaces); + bfill((uchar*) end-spaces,spaces,' '); + } +} + +static void uf_endspace(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, uchar *to, + uchar *end) +{ + uint spaces; + if ((spaces=get_bits(bit_buff,rec->space_length_bits))+to > end) + { + bit_buff->error=1; + return; + } + if (to+spaces != end) + decode_bytes(rec,bit_buff,to,end-spaces); + bfill((uchar*) end-spaces,spaces,' '); +} + +static void uf_space_prespace_selected(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end) +{ + uint spaces; + if (get_bit(bit_buff)) + bfill((uchar*) to,(end-to),' '); + else + { + if (get_bit(bit_buff)) + { + if ((spaces=get_bits(bit_buff,rec->space_length_bits))+to > end) + { + bit_buff->error=1; + return; + } + bfill((uchar*) to,spaces,' '); + if (to+spaces != end) + decode_bytes(rec,bit_buff,to+spaces,end); + } + else + decode_bytes(rec,bit_buff,to,end); + } +} + + +static void uf_prespace_selected(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end) +{ + uint spaces; + if (get_bit(bit_buff)) + { + if ((spaces=get_bits(bit_buff,rec->space_length_bits))+to > end) + { + bit_buff->error=1; + return; + } + bfill((uchar*) to,spaces,' '); + if (to+spaces != end) + decode_bytes(rec,bit_buff,to+spaces,end); + } + else + decode_bytes(rec,bit_buff,to,end); +} + + +static void uf_space_prespace(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, uchar *to, + uchar *end) +{ + uint spaces; + if (get_bit(bit_buff)) + bfill((uchar*) to,(end-to),' '); + else + { + if ((spaces=get_bits(bit_buff,rec->space_length_bits))+to > end) + { + bit_buff->error=1; + return; + } + bfill((uchar*) to,spaces,' '); + if (to+spaces != end) + decode_bytes(rec,bit_buff,to+spaces,end); + } +} + +static void uf_prespace(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, uchar *to, + uchar *end) +{ + uint spaces; + if ((spaces=get_bits(bit_buff,rec->space_length_bits))+to > end) + { + bit_buff->error=1; + return; + } + bfill((uchar*) to,spaces,' '); + if (to+spaces != end) + decode_bytes(rec,bit_buff,to+spaces,end); +} + +static void uf_zerofill_normal(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, uchar *to, + uchar *end) +{ + end-=rec->space_length_bits; + decode_bytes(rec,bit_buff,(uchar*) to,end); + bzero((char*) end,rec->space_length_bits); +} + +static void uf_constant(MI_COLUMNDEF *rec, + MI_BIT_BUFF *bit_buff __attribute__((unused)), + uchar *to, + uchar *end) +{ + memcpy(to,rec->huff_tree->intervalls,(size_t) (end-to)); +} + +static void uf_intervall(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, uchar *to, + uchar *end) +{ + reg1 uint field_length=(uint) (end-to); + memcpy(to,rec->huff_tree->intervalls+field_length*decode_pos(bit_buff, + rec->huff_tree), + (size_t) field_length); +} + + +/*ARGSUSED*/ +static void uf_zero(MI_COLUMNDEF *rec __attribute__((unused)), + MI_BIT_BUFF *bit_buff __attribute__((unused)), + uchar *to, uchar *end) +{ + bzero((char*) to,(uint) (end-to)); +} + +static void uf_blob(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end) +{ + if (get_bit(bit_buff)) + bzero((uchar*) to,(end-to)); + else + { + ulong length=get_bits(bit_buff,rec->space_length_bits); + uint pack_length=(uint) (end-to)-portable_sizeof_char_ptr; + if (bit_buff->blob_pos+length > bit_buff->blob_end) + { + bit_buff->error=1; + bzero((uchar*) to,(end-to)); + return; + } + decode_bytes(rec,bit_buff,bit_buff->blob_pos,bit_buff->blob_pos+length); + _mi_store_blob_length((uchar*) to,pack_length,length); + memcpy(to+pack_length, &bit_buff->blob_pos, sizeof(char*)); + bit_buff->blob_pos+=length; + } +} + + +static void uf_varchar1(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end __attribute__((unused))) +{ + if (get_bit(bit_buff)) + to[0]= 0; /* Zero lengths */ + else + { + ulong length=get_bits(bit_buff,rec->space_length_bits); + *to= (uchar) length; + decode_bytes(rec,bit_buff,to+1,to+1+length); + } +} + + +static void uf_varchar2(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, + uchar *to, uchar *end __attribute__((unused))) +{ + if (get_bit(bit_buff)) + to[0]=to[1]=0; /* Zero lengths */ + else + { + ulong length=get_bits(bit_buff,rec->space_length_bits); + int2store(to,length); + decode_bytes(rec,bit_buff,to+2,to+2+length); + } +} + + /* Functions to decode of buffer of bits */ + +#if BITS_SAVED == 64 + +static void decode_bytes(MI_COLUMNDEF *rec,MI_BIT_BUFF *bit_buff,uchar *to, + uchar *end) +{ + reg1 uint bits,low_byte; + reg3 uint16 *pos; + reg4 uint table_bits,table_and; + MI_DECODE_TREE *decode_tree; + + decode_tree=rec->decode_tree; + bits=bit_buff->bits; /* Save in reg for quicker access */ + table_bits=decode_tree->quick_table_bits; + table_and= (1 << table_bits)-1; + + do + { + if (bits <= 32) + { + if (bit_buff->pos > bit_buff->end+4) + { + bit_buff->error=1; + return; /* Can't be right */ + } + bit_buff->current_byte= (bit_buff->current_byte << 32) | + ((((uint) bit_buff->pos[3])) | + (((uint) bit_buff->pos[2]) << 8) | + (((uint) bit_buff->pos[1]) << 16) | + (((uint) bit_buff->pos[0]) << 24)); + bit_buff->pos+=4; + bits+=32; + } + /* + First use info in quick_table. + + The quick table is an array of 16-bit values. There exists one + value for each possible code representable by table_bits bits. + In most cases table_bits is 9. So there are 512 16-bit values. + + If the high-order bit (16) is set (IS_CHAR) then the array slot + for this value is a valid Huffman code for a resulting byte value. + + The low-order 8 bits (1..8) are the resulting byte value. + + Bits 9..14 are the length of the Huffman code for this byte value. + This means so many bits from the input stream were needed to + represent this byte value. The remaining bits belong to later + Huffman codes. This also means that for every Huffman code shorter + than table_bits there are multiple entires in the array, which + differ just in the unused bits. + + If the high-order bit (16) is clear (0) then the remaining bits are + the position of the remaining Huffman decode tree segment behind the + quick table. + */ + low_byte=(uint) (bit_buff->current_byte >> (bits - table_bits)) & table_and; + low_byte=decode_tree->table[low_byte]; + if (low_byte & IS_CHAR) + { + /* + All Huffman codes of less or equal table_bits length are in the + quick table. This is one of them. + */ + *to++ = (low_byte & 255); /* Found char in quick table */ + bits-= ((low_byte >> 8) & 31); /* Remove bits used */ + } + else + { /* Map through rest of decode-table */ + /* This means that the Huffman code must be longer than table_bits. */ + pos=decode_tree->table+low_byte; + bits-=table_bits; + /* NOTE: decode_bytes_test_bit() is a macro which contains a break !!! */ + for (;;) + { + low_byte=(uint) (bit_buff->current_byte >> (bits-8)); + decode_bytes_test_bit(0); + decode_bytes_test_bit(1); + decode_bytes_test_bit(2); + decode_bytes_test_bit(3); + decode_bytes_test_bit(4); + decode_bytes_test_bit(5); + decode_bytes_test_bit(6); + decode_bytes_test_bit(7); + bits-=8; + } + *to++ = *pos; + } + } while (to != end); + + bit_buff->bits=bits; + return; +} + +#else + +static void decode_bytes(MI_COLUMNDEF *rec, MI_BIT_BUFF *bit_buff, uchar *to, + uchar *end) +{ + reg1 uint bits,low_byte; + reg3 uint16 *pos; + reg4 uint table_bits,table_and; + MI_DECODE_TREE *decode_tree; + + decode_tree=rec->huff_tree; + bits=bit_buff->bits; /* Save in reg for quicker access */ + table_bits=decode_tree->quick_table_bits; + table_and= (1 << table_bits)-1; + + do + { + if (bits < table_bits) + { + if (bit_buff->pos > bit_buff->end+1) + { + bit_buff->error=1; + return; /* Can't be right */ + } +#if BITS_SAVED == 32 + bit_buff->current_byte= (bit_buff->current_byte << 24) | + (((uint) ((uchar) bit_buff->pos[2]))) | + (((uint) ((uchar) bit_buff->pos[1])) << 8) | + (((uint) ((uchar) bit_buff->pos[0])) << 16); + bit_buff->pos+=3; + bits+=24; +#else + if (bits) /* We must have at leasts 9 bits */ + { + bit_buff->current_byte= (bit_buff->current_byte << 8) | + (uint) ((uchar) bit_buff->pos[0]); + bit_buff->pos++; + bits+=8; + } + else + { + bit_buff->current_byte= ((uint) ((uchar) bit_buff->pos[0]) << 8) | + ((uint) ((uchar) bit_buff->pos[1])); + bit_buff->pos+=2; + bits+=16; + } +#endif + } + /* First use info in quick_table */ + low_byte=(bit_buff->current_byte >> (bits - table_bits)) & table_and; + low_byte=decode_tree->table[low_byte]; + if (low_byte & IS_CHAR) + { + *to++ = (low_byte & 255); /* Found char in quick table */ + bits-= ((low_byte >> 8) & 31); /* Remove bits used */ + } + else + { /* Map through rest of decode-table */ + pos=decode_tree->table+low_byte; + bits-=table_bits; + for (;;) + { + if (bits < 8) + { /* We don't need to check end */ +#if BITS_SAVED == 32 + bit_buff->current_byte= (bit_buff->current_byte << 24) | + (((uint) ((uchar) bit_buff->pos[2]))) | + (((uint) ((uchar) bit_buff->pos[1])) << 8) | + (((uint) ((uchar) bit_buff->pos[0])) << 16); + bit_buff->pos+=3; + bits+=24; +#else + bit_buff->current_byte= (bit_buff->current_byte << 8) | + (uint) ((uchar) bit_buff->pos[0]); + bit_buff->pos+=1; + bits+=8; +#endif + } + low_byte=(uint) (bit_buff->current_byte >> (bits-8)); + decode_bytes_test_bit(0); + decode_bytes_test_bit(1); + decode_bytes_test_bit(2); + decode_bytes_test_bit(3); + decode_bytes_test_bit(4); + decode_bytes_test_bit(5); + decode_bytes_test_bit(6); + decode_bytes_test_bit(7); + bits-=8; + } + *to++ = (uchar) *pos; + } + } while (to != end); + + bit_buff->bits=bits; + return; +} +#endif /* BIT_SAVED == 64 */ + + +static uint decode_pos(MI_BIT_BUFF *bit_buff, MI_DECODE_TREE *decode_tree) +{ + uint16 *pos=decode_tree->table; + for (;;) + { + if (get_bit(bit_buff)) + pos++; + if (*pos & IS_CHAR) + return (uint) (*pos & ~IS_CHAR); + pos+= *pos; + } +} + + +int _mi_read_rnd_pack_record(MI_INFO *info, uchar *buf, + register my_off_t filepos, + my_bool skip_deleted_blocks) +{ + uint b_type; + MI_BLOCK_INFO block_info; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("_mi_read_rnd_pack_record"); + + if (filepos >= info->state->data_file_length) + { + my_errno= HA_ERR_END_OF_FILE; + goto err; + } + + if (info->opt_flag & READ_CACHE_USED) + { + if (_mi_read_cache(&info->rec_cache, (uchar*) block_info.header, + filepos, share->pack.ref_length, + skip_deleted_blocks ? READING_NEXT : 0)) + goto err; + b_type=_mi_pack_get_block_info(info, &info->bit_buff, &block_info, + &info->rec_buff, -1, filepos); + } + else + b_type=_mi_pack_get_block_info(info, &info->bit_buff, &block_info, + &info->rec_buff, info->dfile, filepos); + if (b_type) + goto err; /* Error code is already set */ +#ifndef DBUG_OFF + if (block_info.rec_len > share->max_pack_length) + { + my_errno=HA_ERR_WRONG_IN_RECORD; + goto err; + } +#endif + + if (info->opt_flag & READ_CACHE_USED) + { + if (_mi_read_cache(&info->rec_cache, (uchar*) info->rec_buff, + block_info.filepos, block_info.rec_len, + skip_deleted_blocks ? READING_NEXT : 0)) + goto err; + } + else + { + if (mysql_file_read(info->dfile, + (uchar*) info->rec_buff + block_info.offset, + block_info.rec_len-block_info.offset, MYF(MY_NABP))) + goto err; + } + info->packed_length=block_info.rec_len; + info->lastpos=filepos; + info->nextpos=block_info.filepos+block_info.rec_len; + info->update|= HA_STATE_AKTIV | HA_STATE_KEY_CHANGED; + + info->rec_buff[block_info.rec_len]= 0; /* Keep valgrind happy */ + DBUG_RETURN(_mi_pack_rec_unpack(info, &info->bit_buff, buf, + info->rec_buff, block_info.rec_len)); + err: + DBUG_RETURN(my_errno); +} + + + /* Read and process header from a huff-record-file */ + +uint _mi_pack_get_block_info(MI_INFO *myisam, MI_BIT_BUFF *bit_buff, + MI_BLOCK_INFO *info, uchar **rec_buff_p, + File file, my_off_t filepos) +{ + uchar *header=info->header; + uint head_length, UNINIT_VAR(ref_length); + + if (file >= 0) + { + ref_length=myisam->s->pack.ref_length; + /* + We can't use mysql_file_pread() here because mi_read_rnd_pack_record + assumes position is ok + */ + mysql_file_seek(file, filepos, MY_SEEK_SET, MYF(0)); + if (mysql_file_read(file, header, ref_length, MYF(MY_NABP))) + return BLOCK_FATAL_ERROR; + DBUG_DUMP("header",(uchar*) header,ref_length); + } + head_length= read_pack_length((uint) myisam->s->pack.version, header, + &info->rec_len); + if (myisam->s->base.blobs) + { + head_length+= read_pack_length((uint) myisam->s->pack.version, + header + head_length, &info->blob_len); + /* + Ensure that the record buffer is big enough for the compressed + record plus all expanded blobs. [We do not have an extra buffer + for the resulting blobs. Sigh.] + */ + if (!(mi_alloc_rec_buff(myisam,info->rec_len + info->blob_len, + rec_buff_p))) + return BLOCK_FATAL_ERROR; /* not enough memory */ + bit_buff->blob_pos= (uchar*) *rec_buff_p + info->rec_len; + bit_buff->blob_end= bit_buff->blob_pos + info->blob_len; + myisam->blob_length=info->blob_len; + } + info->filepos=filepos+head_length; + if (file > 0) + { + info->offset=MY_MIN(info->rec_len, ref_length - head_length); + memcpy(*rec_buff_p, header + head_length, info->offset); + } + return 0; +} + + +/* + Rutines for bit buffer + Note: buffer must be 6 byte bigger than longest row +*/ + +static void init_bit_buffer(MI_BIT_BUFF *bit_buff, uchar *buffer, uint length) +{ + bit_buff->pos=buffer; + bit_buff->end=buffer+length; + bit_buff->bits=bit_buff->error=0; + bit_buff->current_byte=0; /* Avoid valgrind errors */ +} + +static uint fill_and_get_bits(MI_BIT_BUFF *bit_buff, uint count) +{ + uint tmp; + count-=bit_buff->bits; + tmp=(bit_buff->current_byte & mask[bit_buff->bits]) << count; + fill_buffer(bit_buff); + bit_buff->bits=BITS_SAVED - count; + return tmp+(bit_buff->current_byte >> (BITS_SAVED - count)); +} + + /* Fill in empty bit_buff->current_byte from buffer */ + /* Sets bit_buff->error if buffer is exhausted */ + +static void fill_buffer(MI_BIT_BUFF *bit_buff) +{ + if (bit_buff->pos >= bit_buff->end) + { + bit_buff->error= 1; + bit_buff->current_byte=0; + return; + } + +#if BITS_SAVED == 64 + bit_buff->current_byte= ((((uint) ((uchar) bit_buff->pos[7]))) | + (((uint) ((uchar) bit_buff->pos[6])) << 8) | + (((uint) ((uchar) bit_buff->pos[5])) << 16) | + (((uint) ((uchar) bit_buff->pos[4])) << 24) | + ((ulonglong) + ((((uint) ((uchar) bit_buff->pos[3]))) | + (((uint) ((uchar) bit_buff->pos[2])) << 8) | + (((uint) ((uchar) bit_buff->pos[1])) << 16) | + (((uint) ((uchar) bit_buff->pos[0])) << 24)) << 32)); + bit_buff->pos+=8; +#else +#if BITS_SAVED == 32 + bit_buff->current_byte= (((uint) ((uchar) bit_buff->pos[3])) | + (((uint) ((uchar) bit_buff->pos[2])) << 8) | + (((uint) ((uchar) bit_buff->pos[1])) << 16) | + (((uint) ((uchar) bit_buff->pos[0])) << 24)); + bit_buff->pos+=4; +#else + bit_buff->current_byte= (uint) (((uint) ((uchar) bit_buff->pos[1])) | + (((uint) ((uchar) bit_buff->pos[0])) << 8)); + bit_buff->pos+=2; +#endif +#endif +} + + /* Get number of bits neaded to represent value */ + +static uint max_bit(register uint value) +{ + reg2 uint power=1; + + while ((value>>=1)) + power++; + return (power); +} + + +/***************************************************************************** + Some redefined functions to handle files when we are using memmap +*****************************************************************************/ + +#ifdef HAVE_MMAP + +static int _mi_read_mempack_record(MI_INFO *info,my_off_t filepos,uchar *buf); +static int _mi_read_rnd_mempack_record(MI_INFO*, uchar *,my_off_t, my_bool); + +my_bool _mi_memmap_file(MI_INFO *info) +{ + MYISAM_SHARE *share=info->s; + my_bool eom; + + DBUG_ENTER("mi_memmap_file"); + + if (!info->s->file_map) + { + my_off_t data_file_length= share->state.state.data_file_length; + + if (myisam_mmap_size != SIZE_T_MAX) + { + mysql_mutex_lock(&THR_LOCK_myisam_mmap); + eom= data_file_length > myisam_mmap_size - myisam_mmap_used - MEMMAP_EXTRA_MARGIN; + if (!eom) + myisam_mmap_used+= data_file_length + MEMMAP_EXTRA_MARGIN; + mysql_mutex_unlock(&THR_LOCK_myisam_mmap); + } + else + eom= data_file_length > myisam_mmap_size - MEMMAP_EXTRA_MARGIN; + + if (eom) + { + DBUG_PRINT("warning", ("File is too large for mmap")); + DBUG_RETURN(0); + } + if (mysql_file_seek(info->dfile, 0L, MY_SEEK_END, MYF(0)) < + share->state.state.data_file_length+MEMMAP_EXTRA_MARGIN) + { + DBUG_PRINT("warning",("File isn't extended for memmap")); + if (myisam_mmap_size != SIZE_T_MAX) + { + mysql_mutex_lock(&THR_LOCK_myisam_mmap); + myisam_mmap_used-= data_file_length + MEMMAP_EXTRA_MARGIN; + mysql_mutex_unlock(&THR_LOCK_myisam_mmap); + } + DBUG_RETURN(0); + } + if (mi_dynmap_file(info, + share->state.state.data_file_length + + MEMMAP_EXTRA_MARGIN)) + { + if (myisam_mmap_size != SIZE_T_MAX) + { + mysql_mutex_lock(&THR_LOCK_myisam_mmap); + myisam_mmap_used-= data_file_length + MEMMAP_EXTRA_MARGIN; + mysql_mutex_unlock(&THR_LOCK_myisam_mmap); + } + DBUG_RETURN(0); + } + } + info->opt_flag|= MEMMAP_USED; + info->read_record= share->read_record= _mi_read_mempack_record; + share->read_rnd= _mi_read_rnd_mempack_record; + DBUG_RETURN(1); +} + + +void _mi_unmap_file(MI_INFO *info) +{ + DBUG_ASSERT(info->s->options & HA_OPTION_COMPRESS_RECORD); + + (void) my_munmap((char*) info->s->file_map, info->s->mmaped_length); + + if (myisam_mmap_size != SIZE_T_MAX) + { + mysql_mutex_lock(&THR_LOCK_myisam_mmap); + myisam_mmap_used-= info->s->mmaped_length; + mysql_mutex_unlock(&THR_LOCK_myisam_mmap); + } +} + + +static uchar *_mi_mempack_get_block_info(MI_INFO *myisam, + MI_BIT_BUFF *bit_buff, + MI_BLOCK_INFO *info, + uchar **rec_buff_p, + uchar *header) +{ + header+= read_pack_length((uint) myisam->s->pack.version, header, + &info->rec_len); + if (myisam->s->base.blobs) + { + header+= read_pack_length((uint) myisam->s->pack.version, header, + &info->blob_len); + /* mi_alloc_rec_buff sets my_errno on error */ + if (!(mi_alloc_rec_buff(myisam, info->blob_len , + rec_buff_p))) + return 0; /* not enough memory */ + bit_buff->blob_pos= (uchar*) *rec_buff_p; + bit_buff->blob_end= (uchar*) *rec_buff_p + info->blob_len; + } + return header; +} + + +static int _mi_read_mempack_record(MI_INFO *info, my_off_t filepos, uchar *buf) +{ + MI_BLOCK_INFO block_info; + MYISAM_SHARE *share=info->s; + uchar *pos; + DBUG_ENTER("mi_read_mempack_record"); + + if (filepos == HA_OFFSET_ERROR) + DBUG_RETURN(-1); /* _search() didn't find record */ + + if (!(pos= (uchar*) _mi_mempack_get_block_info(info, &info->bit_buff, + &block_info, &info->rec_buff, + (uchar*) share->file_map+ + filepos))) + DBUG_RETURN(-1); + /* No need to end-zero pos here for valgrind as data is memory mapped */ + DBUG_RETURN(_mi_pack_rec_unpack(info, &info->bit_buff, buf, + pos, block_info.rec_len)); +} + + +/*ARGSUSED*/ +static int _mi_read_rnd_mempack_record(MI_INFO *info, uchar *buf, + register my_off_t filepos, + my_bool skip_deleted_blocks + __attribute__((unused))) +{ + MI_BLOCK_INFO block_info; + MYISAM_SHARE *share=info->s; + uchar *pos,*start; + DBUG_ENTER("_mi_read_rnd_mempack_record"); + + if (filepos >= share->state.state.data_file_length) + { + my_errno=HA_ERR_END_OF_FILE; + goto err; + } + if (!(pos= (uchar*) _mi_mempack_get_block_info(info, &info->bit_buff, + &block_info, &info->rec_buff, + (uchar*) + (start=share->file_map+ + filepos)))) + goto err; +#ifndef DBUG_OFF + if (block_info.rec_len > info->s->max_pack_length) + { + my_errno=HA_ERR_WRONG_IN_RECORD; + goto err; + } +#endif + info->packed_length=block_info.rec_len; + info->lastpos=filepos; + info->nextpos=filepos+(uint) (pos-start)+block_info.rec_len; + info->update|= HA_STATE_AKTIV | HA_STATE_KEY_CHANGED; + + DBUG_RETURN (_mi_pack_rec_unpack(info, &info->bit_buff, buf, + pos, block_info.rec_len)); + err: + DBUG_RETURN(my_errno); +} + +#endif /* HAVE_MMAP */ + + /* Save length of row */ + +uint save_pack_length(uint version, uchar *block_buff, ulong length) +{ + if (length < 254) + { + *(uchar*) block_buff= (uchar) length; + return 1; + } + if (length <= 65535) + { + *(uchar*) block_buff=254; + int2store(block_buff+1,(uint) length); + return 3; + } + *(uchar*) block_buff=255; + if (version == 1) /* old format */ + { + DBUG_ASSERT(length <= 0xFFFFFF); + int3store(block_buff + 1, (ulong) length); + return 4; + } + else + { + int4store(block_buff + 1, (ulong) length); + return 5; + } +} + + +static uint read_pack_length(uint version, const uchar *buf, ulong *length) +{ + if (buf[0] < 254) + { + *length= buf[0]; + return 1; + } + else if (buf[0] == 254) + { + *length= uint2korr(buf + 1); + return 3; + } + if (version == 1) /* old format */ + { + *length= uint3korr(buf + 1); + return 4; + } + else + { + *length= uint4korr(buf + 1); + return 5; + } +} + + +uint calc_pack_length(uint version, ulong length) +{ + return (length < 254) ? 1 : (length < 65536) ? 3 : (version == 1) ? 4 : 5; +} diff --git a/storage/myisam/mi_page.c b/storage/myisam/mi_page.c new file mode 100644 index 00000000..4d199f9e --- /dev/null +++ b/storage/myisam/mi_page.c @@ -0,0 +1,156 @@ +/* + Copyright (c) 2000, 2010, Oracle and/or its affiliates + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Read and write key blocks */ + +#include "myisamdef.h" + + /* Fetch a key-page in memory */ + +uchar *_mi_fetch_keypage(register MI_INFO *info, MI_KEYDEF *keyinfo, + my_off_t page, int level, + uchar *buff, int return_buffer) +{ + uchar *tmp; + uint page_size; + DBUG_ENTER("_mi_fetch_keypage"); + DBUG_PRINT("enter",("page: %ld", (long) page)); + + tmp=(uchar*) key_cache_read(info->s->key_cache, + info->s->kfile, page, level, (uchar*) buff, + (uint) keyinfo->block_length, + (uint) keyinfo->block_length, + return_buffer); + if (tmp == info->buff) + info->buff_used=1; + else if (!tmp) + { + DBUG_PRINT("error",("Got errno: %d from key_cache_read",my_errno)); + info->last_keypage=HA_OFFSET_ERROR; + mi_print_error(info->s, HA_ERR_CRASHED); + my_errno=HA_ERR_CRASHED; + DBUG_RETURN(0); + } + info->last_keypage=page; + page_size=mi_getint(tmp); + if (page_size < 4 || page_size > keyinfo->block_length) + { + DBUG_PRINT("error",("page %lu had wrong page length: %u", + (ulong) page, page_size)); + DBUG_DUMP("page", tmp, keyinfo->block_length); + info->last_keypage = HA_OFFSET_ERROR; + mi_print_error(info->s, HA_ERR_CRASHED); + my_errno = HA_ERR_CRASHED; + tmp = 0; + } + DBUG_RETURN(tmp); +} /* _mi_fetch_keypage */ + + + /* Write a key-page on disk */ + +int _mi_write_keypage(register MI_INFO *info, register MI_KEYDEF *keyinfo, + my_off_t page, int level, uchar *buff) +{ + reg3 uint length; + DBUG_ENTER("_mi_write_keypage"); + +#ifndef FAST /* Safety check */ + if (page < info->s->base.keystart || + page+keyinfo->block_length > info->state->key_file_length || + (page & (MI_MIN_KEY_BLOCK_LENGTH-1))) + { + DBUG_PRINT("error",("Trying to write inside key status region: key_start: %lu length: %lu page: %lu", + (long) info->s->base.keystart, + (long) info->state->key_file_length, + (long) page)); + my_errno=EINVAL; + DBUG_RETURN((-1)); + } + DBUG_PRINT("page",("write page at: %lu",(long) page)); + DBUG_DUMP("buff",(uchar*) buff,mi_getint(buff)); +#endif + + if ((length=keyinfo->block_length) > IO_SIZE*2 && + info->state->key_file_length != page+length) + length= ((mi_getint(buff)+IO_SIZE-1) & (uint) ~(IO_SIZE-1)); + DBUG_RETURN((key_cache_write(info->s->key_cache, + info->s->kfile, &info->s->dirty_part_map, + page, level, (uchar*) buff, length, + (uint) keyinfo->block_length, + (int) ((info->lock_type != F_UNLCK) || + info->s->delay_key_write)))); +} /* mi_write_keypage */ + + + /* Remove page from disk */ + +int _mi_dispose(register MI_INFO *info, MI_KEYDEF *keyinfo, my_off_t pos, + int level) +{ + my_off_t old_link; + uchar buff[8]; + DBUG_ENTER("_mi_dispose"); + DBUG_PRINT("enter",("pos: %ld", (long) pos)); + + old_link= info->s->state.key_del[keyinfo->block_size_index]; + info->s->state.key_del[keyinfo->block_size_index]= pos; + mi_sizestore(buff,old_link); + info->s->state.changed|= STATE_NOT_SORTED_PAGES; + DBUG_RETURN(key_cache_write(info->s->key_cache, + info->s->kfile, &info->s->dirty_part_map, + pos , level, buff, + sizeof(buff), + (uint) keyinfo->block_length, + (int) (info->lock_type != F_UNLCK))); +} /* _mi_dispose */ + + + /* Make new page on disk */ + +my_off_t _mi_new(register MI_INFO *info, MI_KEYDEF *keyinfo, int level) +{ + my_off_t pos; + uchar buff[8]; + DBUG_ENTER("_mi_new"); + + if ((pos= info->s->state.key_del[keyinfo->block_size_index]) == + HA_OFFSET_ERROR) + { + if (info->state->key_file_length >= + info->s->base.max_key_file_length - keyinfo->block_length) + { + my_errno=HA_ERR_INDEX_FILE_FULL; + DBUG_RETURN(HA_OFFSET_ERROR); + } + pos=info->state->key_file_length; + info->state->key_file_length+= keyinfo->block_length; + } + else + { + if (!key_cache_read(info->s->key_cache, + info->s->kfile, pos, level, + buff, + (uint) sizeof(buff), + (uint) keyinfo->block_length,0)) + pos= HA_OFFSET_ERROR; + else + info->s->state.key_del[keyinfo->block_size_index]= mi_sizekorr(buff); + } + info->s->state.changed|= STATE_NOT_SORTED_PAGES; + DBUG_PRINT("exit",("Pos: %ld",(long) pos)); + DBUG_RETURN(pos); +} /* _mi_new */ diff --git a/storage/myisam/mi_panic.c b/storage/myisam/mi_panic.c new file mode 100644 index 00000000..541cf399 --- /dev/null +++ b/storage/myisam/mi_panic.c @@ -0,0 +1,119 @@ +/* Copyright (c) 2000, 2003, 2005, 2006 MySQL AB, 2009 Sun Microsystems, Inc. + Use is subject to license terms. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "fulltext.h" + + /* if flag == HA_PANIC_CLOSE then all misam files are closed */ + /* if flag == HA_PANIC_WRITE then all misam files are unlocked and + all changed data in single user misam is written to file */ + /* if flag == HA_PANIC_READ then all misam files that was locked when + mi_panic(HA_PANIC_WRITE) was done is locked. A mi_readinfo() is + done for all single user files to get changes in database */ + + +int mi_panic(enum ha_panic_function flag) +{ + int error=0; + LIST *list_element,*next_open; + MI_INFO *info; + DBUG_ENTER("mi_panic"); + + mysql_mutex_lock(&THR_LOCK_myisam); + for (list_element=myisam_open_list ; list_element ; list_element=next_open) + { + next_open=list_element->next; /* Save if close */ + info=(MI_INFO*) list_element->data; + switch (flag) { + case HA_PANIC_CLOSE: + mysql_mutex_unlock(&THR_LOCK_myisam); /* Not exactly right... */ + if (mi_close(info)) + error=my_errno; + mysql_mutex_lock(&THR_LOCK_myisam); + break; + case HA_PANIC_WRITE: /* Do this to free databases */ +#ifdef CANT_OPEN_FILES_TWICE + if (info->s->options & HA_OPTION_READ_ONLY_DATA) + break; +#endif + if (flush_key_blocks(info->s->key_cache, info->s->kfile, + &info->s->dirty_part_map, FLUSH_RELEASE)) + error=my_errno; + if (info->opt_flag & WRITE_CACHE_USED) + if (flush_io_cache(&info->rec_cache)) + error=my_errno; + if (info->opt_flag & READ_CACHE_USED) + { + if (flush_io_cache(&info->rec_cache)) + error=my_errno; + reinit_io_cache(&info->rec_cache,READ_CACHE,0, + (pbool) (info->lock_type != F_UNLCK),1); + } + if (info->lock_type != F_UNLCK && ! info->was_locked) + { + info->was_locked=info->lock_type; + if (mi_lock_database(info,F_UNLCK)) + error=my_errno; + } +#ifdef CANT_OPEN_FILES_TWICE + if (info->s->kfile >= 0 && mysql_file_close(info->s->kfile, MYF(0))) + error = my_errno; + if (info->dfile >= 0 && mysql_file_close(info->dfile, MYF(0))) + error = my_errno; + info->s->kfile=info->dfile= -1; /* Files aren't open anymore */ +#endif + break; + case HA_PANIC_READ: /* Restore to before WRITE */ +#ifdef CANT_OPEN_FILES_TWICE + { /* Open closed files */ + char name_buff[FN_REFLEN]; + if (info->s->kfile < 0) + if ((info->s->kfile= mysql_file_open(mi_key_file_kfile, + fn_format(name_buff, + info->filename, "", + N_NAME_IEXT, 4), + info->mode, MYF(MY_WME))) < 0) + error = my_errno; + if (info->dfile < 0) + { + if ((info->dfile= mysql_file_open(mi_key_file_dfile, + fn_format(name_buff, + info->filename, "", + N_NAME_DEXT, 4), + info->mode, MYF(MY_WME))) < 0) + error = my_errno; + info->rec_cache.file=info->dfile; + } + } +#endif + if (info->was_locked) + { + if (mi_lock_database(info, info->was_locked)) + error=my_errno; + info->was_locked=0; + } + break; + } + } + if (flag == HA_PANIC_CLOSE) + { + (void) mi_log(0); /* Close log if neaded */ + ft_free_stopwords(); + } + mysql_mutex_unlock(&THR_LOCK_myisam); + if (!error) + DBUG_RETURN(0); + DBUG_RETURN(my_errno=error); +} /* mi_panic */ diff --git a/storage/myisam/mi_preload.c b/storage/myisam/mi_preload.c new file mode 100644 index 00000000..5f9132ab --- /dev/null +++ b/storage/myisam/mi_preload.c @@ -0,0 +1,128 @@ +/* Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* + Preload indexes into key cache +*/ + +#include "myisamdef.h" + + +/* + Preload pages of the index file for a table into the key cache + + SYNOPSIS + mi_preload() + info open table + map map of indexes to preload into key cache + ignore_leaves only non-leaves pages are to be preloaded + + RETURN VALUE + 0 if a success. error code - otherwise. + + NOTES. + At present pages for all indexes are preloaded. + In future only pages for indexes specified in the key_map parameter + of the table will be preloaded. +*/ + +int mi_preload(MI_INFO *info, ulonglong key_map, my_bool ignore_leaves) +{ + uint i; + size_t length, block_length= 0; + uchar *buff= NULL; + MYISAM_SHARE* share= info->s; + uint keys= share->state.header.keys; + MI_KEYDEF *keyinfo= share->keyinfo; + my_off_t key_file_length= share->state.state.key_file_length; + my_off_t pos= share->base.keystart; + DBUG_ENTER("mi_preload"); + + if (!keys || !mi_is_any_key_active(key_map) || key_file_length == pos) + DBUG_RETURN(0); + + /* Preload into a non initialized key cache should never happen. */ + DBUG_ASSERT(share->key_cache->key_cache_inited); + + block_length= keyinfo[0].block_length; + + if (ignore_leaves) + { + /* Check whether all indexes use the same block size */ + for (i= 1 ; i < keys ; i++) + { + if (keyinfo[i].block_length != block_length) + DBUG_RETURN(my_errno= HA_ERR_NON_UNIQUE_BLOCK_SIZE); + } + } + else + block_length= (size_t)share->key_cache->param_block_size; + + length= info->preload_buff_size/block_length * block_length; + set_if_bigger(length, block_length); + + if (!(buff= (uchar *) my_malloc(mi_key_memory_preload_buffer, length, + MYF(MY_WME)))) + DBUG_RETURN(my_errno= HA_ERR_OUT_OF_MEM); + + if (flush_key_blocks(share->key_cache, share->kfile, &share->dirty_part_map, + FLUSH_RELEASE)) + goto err; + + do + { + /* Read the next block of index file into the preload buffer */ + if ((my_off_t) length > (key_file_length-pos)) + length= (size_t) (key_file_length-pos); + if (mysql_file_pread(share->kfile, (uchar*) buff, length, pos, + MYF(MY_FAE|MY_FNABP))) + goto err; + + if (ignore_leaves) + { + uchar *end= buff+length; + do + { + if (mi_test_if_nod(buff)) + { + if (key_cache_insert(share->key_cache, + share->kfile, pos, DFLT_INIT_HITS, + buff, (uint)block_length)) + goto err; + } + pos+= block_length; + } + while ((buff+= block_length) != end); + buff= end-length; + } + else + { + if (key_cache_insert(share->key_cache, + share->kfile, pos, DFLT_INIT_HITS, + (uchar*) buff, (uint)length)) + goto err; + pos+= length; + } + } + while (pos != key_file_length); + + my_free(buff); + DBUG_RETURN(0); + +err: + my_free(buff); + DBUG_RETURN(my_errno= errno); +} + diff --git a/storage/myisam/mi_range.c b/storage/myisam/mi_range.c new file mode 100644 index 00000000..54350f7a --- /dev/null +++ b/storage/myisam/mi_range.c @@ -0,0 +1,327 @@ +/* + Copyright (c) 2000, 2011, Oracle and/or its affiliates + Copyright (c) 2010, 2020, MariaDB Corporation. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* + Gives a approximated number of how many records there is between two keys. + Used when optimizing querries. + */ + +#include "myisamdef.h" +#include "rt_index.h" + +static double _mi_record_pos(MI_INFO *, const uchar *, key_part_map, + enum ha_rkey_function, ulonglong *); +static double _mi_search_pos(MI_INFO *,MI_KEYDEF *,uchar *, uint,uint, + my_off_t,my_bool, ulonglong *); +static uint _mi_keynr(MI_INFO *info,MI_KEYDEF *,uchar *, uchar *,uint *); + +/* + Estimate how many records there is in a given range + + SYNOPSIS + mi_records_in_range() + info MyISAM handler + inx Index to use + min_key Min key. Is = 0 if no min range + max_key Max key. Is = 0 if no max range + + NOTES + We should ONLY return 0 if there is no rows in range + + RETURN + HA_POS_ERROR error (or we can't estimate number of rows) + number Estimated number of rows +*/ + +ha_rows mi_records_in_range(MI_INFO *info, int inx, + const key_range *min_key, const key_range *max_key, + page_range *pages) +{ + ha_rows res; + double start_pos,end_pos,diff; + DBUG_ENTER("mi_records_in_range"); + + if ((inx = _mi_check_index(info,inx)) < 0) + DBUG_RETURN(HA_POS_ERROR); + + if (fast_mi_readinfo(info)) + DBUG_RETURN(HA_POS_ERROR); + info->update&= (HA_STATE_CHANGED+HA_STATE_ROW_CHANGED); + if (info->s->concurrent_insert) + mysql_rwlock_rdlock(&info->s->key_root_lock[inx]); + + switch(info->s->keyinfo[inx].key_alg){ +#ifdef HAVE_RTREE_KEYS + case HA_KEY_ALG_RTREE: + { + uchar * key_buff; + uint start_key_len; + + /* + The problem is that the optimizer doesn't support + RTree keys properly at the moment. + Hope this will be fixed some day. + But now NULL in the min_key means that we + didn't make the task for the RTree key + and expect BTree functionality from it. + As it's not able to handle such request + we return the error. + */ + if (!min_key) + { + res= HA_POS_ERROR; + break; + } + key_buff= info->lastkey+info->s->base.max_key_length; + start_key_len= _mi_pack_key(info,inx, key_buff, + (uchar*) min_key->key, min_key->keypart_map, + (HA_KEYSEG**) 0); + res= rtree_estimate(info, inx, key_buff, start_key_len, + myisam_read_vec[min_key->flag]); + res= res ? res : 1; /* Don't return 0 */ + break; + } +#endif + case HA_KEY_ALG_BTREE: + default: + start_pos= (min_key ?_mi_record_pos(info, min_key->key, + min_key->keypart_map, min_key->flag, + &pages->first_page) + : (double) 0); + end_pos= (max_key ? _mi_record_pos(info, max_key->key, + max_key->keypart_map, max_key->flag, + &pages->last_page) + : (double) info->state->records); + res= (end_pos < start_pos ? (ha_rows) 0 : + (end_pos == start_pos ? (ha_rows) 1 : (ha_rows) (end_pos-start_pos))); + if (start_pos == (double) HA_POS_ERROR || end_pos == (double) HA_POS_ERROR) + res=HA_POS_ERROR; + else + { + diff= end_pos - start_pos; + if (diff >= 0) + { + if (!(res= (ha_rows) (diff + 0.5))) + res= 1; + } + else + res= 0; + } + } + + if (info->s->concurrent_insert) + mysql_rwlock_unlock(&info->s->key_root_lock[inx]); + fast_mi_writeinfo(info); + + DBUG_PRINT("info",("records: %ld",(ulong) (res))); + DBUG_RETURN(res); +} + + +/* + To find an approximate relative position of a key tuple among all index + key tuples would not be hard if we considered B-trees where all key + tuples were contained only in leaf nodes. If we consider a B-tree where + key tuples are stored also in non-leaf nodes we have to convert such + tree into the tree of the first type. The transformation procedure is + simple: the key tuple k goes alter the last key tuple in the most right + sub-tree pointer to which is coupled with k. As a result of this + transformation each leaf node except the most right one in the tree will + contain one extra key tuple following those originally belonging to + the leaf. +*/ + + +/* Find relative position (in records) for key in index-tree */ + +static double _mi_record_pos(MI_INFO *info, const uchar *key, + key_part_map keypart_map, + enum ha_rkey_function search_flag, + ulonglong *final_page) +{ + uint inx=(uint) info->lastinx, nextflag, key_len; + MI_KEYDEF *keyinfo=info->s->keyinfo+inx; + uchar *key_buff; + double pos; + + DBUG_ENTER("_mi_record_pos"); + DBUG_PRINT("enter",("search_flag: %d",search_flag)); + DBUG_ASSERT(keypart_map); + + key_buff=info->lastkey+info->s->base.max_key_length; + key_len=_mi_pack_key(info,inx,key_buff,(uchar*) key, keypart_map, + (HA_KEYSEG**) 0); + DBUG_EXECUTE("key",_mi_print_key(DBUG_FILE,keyinfo->seg, + (uchar*) key_buff,key_len);); + nextflag=myisam_read_vec[search_flag]; + if (!(nextflag & (SEARCH_FIND | SEARCH_NO_FIND | SEARCH_LAST))) + key_len=USE_WHOLE_KEY; + + /* + my_handler.c:ha_compare_text() has a flag 'skip_end_space'. + This is set in my_handler.c:ha_key_cmp() in dependence on the + compare flags 'nextflag' and the column type. + + TEXT columns are of type HA_KEYTYPE_VARTEXT. In this case the + condition is skip_end_space= ((nextflag & (SEARCH_FIND | + SEARCH_UPDATE)) == SEARCH_FIND). + + SEARCH_FIND is used for an exact key search. The combination + SEARCH_FIND | SEARCH_UPDATE is used in write/update/delete + operations with a comment like "Not real duplicates", whatever this + means. From the condition above we can see that 'skip_end_space' is + always false for these operations. The result is that trailing space + counts in key comparison and hence, empty strings ('', string length + zero, but not NULL) compare less that strings starting with control + characters and these in turn compare less than strings starting with + blanks. + + When estimating the number of records in a key range, we request an + exact search for the minimum key. This translates into a plain + SEARCH_FIND flag. Using this alone would lead to a 'skip_end_space' + compare. Empty strings would be expected above control characters. + Their keys would not be found because they are located below control + characters. + + This is the reason that we add the SEARCH_UPDATE flag here. It makes + the key estimation compare in the same way like key write operations + do. Only so we will find the keys where they have been inserted. + + Adding the flag unconditionally does not hurt as it is used in the + above mentioned condition only. So it can safely be used together + with other flags. + */ + pos=_mi_search_pos(info,keyinfo,key_buff,key_len, + nextflag | SEARCH_SAVE_BUFF | SEARCH_UPDATE, + info->s->state.key_root[inx], TRUE, + final_page); + if (pos >= 0.0) + { + DBUG_PRINT("exit",("pos: %g",(pos*info->state->records))); + DBUG_RETURN(pos*info->state->records); + } + DBUG_RETURN((double) (HA_POS_ERROR)); +} + + + /* This is a modified version of _mi_search */ + /* Returns offset for key in indextable (decimal 0.0 <= x <= 1.0) */ + +static double _mi_search_pos(register MI_INFO *info, + register MI_KEYDEF *keyinfo, + uchar *key, uint key_len, uint nextflag, + register my_off_t pos, my_bool last_in_level, + ulonglong *final_page) +{ + int flag; + uint nod_flag,keynr,UNINIT_VAR(max_keynr); + my_bool after_key; + uchar *keypos,*buff; + double offset; + DBUG_ENTER("_mi_search_pos"); + + if (pos == HA_OFFSET_ERROR) + DBUG_RETURN(0.5); + + if (!(buff=_mi_fetch_keypage(info,keyinfo,pos,DFLT_INIT_HITS,info->buff,1))) + goto err; + *final_page= pos; + flag=(*keyinfo->bin_search)(info,keyinfo,buff,key,key_len,nextflag, + &keypos,info->lastkey, &after_key); + nod_flag=mi_test_if_nod(buff); + keynr=_mi_keynr(info,keyinfo,buff,keypos,&max_keynr); + + if (flag) + { + if (flag == MI_FOUND_WRONG_KEY) + DBUG_RETURN(-1); /* error */ + /* + Didn't found match. keypos points at next (bigger) key + Try to find a smaller, better matching key. + Matches keynr + [0-1] + */ + if (flag > 0 && ! nod_flag) + offset= 1.0; + else if ((offset=_mi_search_pos(info,keyinfo,key,key_len,nextflag, + _mi_kpos(nod_flag,keypos), + last_in_level && after_key, + final_page)) < 0) + DBUG_RETURN(offset); + } + else + { + /* + Found match. Keypos points at the start of the found key + Matches keynr+1 + */ + offset=1.0; /* Matches keynr+1 */ + if ((nextflag & SEARCH_FIND) && nod_flag && + ((keyinfo->flag & (HA_NOSAME | HA_NULL_PART)) != HA_NOSAME || + key_len != USE_WHOLE_KEY)) + { + /* + There may be identical keys in the tree. Try to match on of those. + Matches keynr + [0-1] + */ + if ((offset=_mi_search_pos(info,keyinfo,key,key_len,SEARCH_FIND, + _mi_kpos(nod_flag,keypos), + last_in_level && after_key, + final_page)) < 0) + DBUG_RETURN(offset); /* Read error */ + } + } + DBUG_PRINT("info",("keynr: %d offset: %g max_keynr: %d nod: %d flag: %d", + keynr,offset,max_keynr,nod_flag,flag)); + DBUG_RETURN((keynr+offset-MY_TEST(!nod_flag))/ + (max_keynr+MY_TEST(nod_flag || !last_in_level))); +err: + DBUG_PRINT("exit",("Error: %d",my_errno)); + DBUG_RETURN (-1.0); +} + + + /* Get keynummer of current key and max number of keys in nod */ + +static uint _mi_keynr(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page, + uchar *keypos, uint *ret_max_key) +{ + uint nod_flag,keynr,max_key; + uchar t_buff[HA_MAX_KEY_BUFF],*end; + + end= page+mi_getint(page); + nod_flag=mi_test_if_nod(page); + page+=2+nod_flag; + + if (!(keyinfo->flag & (HA_VAR_LENGTH_KEY | HA_BINARY_PACK_KEY))) + { + *ret_max_key= (uint) (end-page)/(keyinfo->keylength+nod_flag); + return (uint) (keypos-page)/(keyinfo->keylength+nod_flag); + } + + max_key=keynr=0; + t_buff[0]=0; /* Safety */ + while (page < end) + { + if (!(*keyinfo->get_key)(keyinfo,nod_flag,&page,t_buff)) + return 0; /* Error */ + max_key++; + if (page == keypos) + keynr=max_key; + } + *ret_max_key=max_key; + return(keynr); +} diff --git a/storage/myisam/mi_rename.c b/storage/myisam/mi_rename.c new file mode 100644 index 00000000..13cf8c60 --- /dev/null +++ b/storage/myisam/mi_rename.c @@ -0,0 +1,45 @@ +/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* + Rename a table +*/ + +#include "fulltext.h" + +int mi_rename(const char *old_name, const char *new_name) +{ + char from[FN_REFLEN],to[FN_REFLEN]; + int save_errno= 0; + DBUG_ENTER("mi_rename"); + +#ifdef EXTRA_DEBUG + check_table_is_closed(old_name,"rename old_table"); + check_table_is_closed(new_name,"rename new table2"); +#endif + + fn_format(from,old_name,"",MI_NAME_IEXT,MY_UNPACK_FILENAME|MY_APPEND_EXT); + fn_format(to,new_name,"",MI_NAME_IEXT,MY_UNPACK_FILENAME|MY_APPEND_EXT); + if (mysql_file_rename_with_symlink(mi_key_file_kfile, from, to, MYF(MY_WME))) + save_errno= my_errno; + fn_format(from,old_name,"",MI_NAME_DEXT,MY_UNPACK_FILENAME|MY_APPEND_EXT); + fn_format(to,new_name,"",MI_NAME_DEXT,MY_UNPACK_FILENAME|MY_APPEND_EXT); + if (mysql_file_rename_with_symlink(mi_key_file_dfile, + from, to, + MYF(MY_WME))) + if (save_errno) + save_errno= my_errno; + DBUG_RETURN(save_errno); +} diff --git a/storage/myisam/mi_rfirst.c b/storage/myisam/mi_rfirst.c new file mode 100644 index 00000000..26f6921b --- /dev/null +++ b/storage/myisam/mi_rfirst.c @@ -0,0 +1,27 @@ +/* Copyright (c) 2000, 2001, 2005-2007 MySQL AB + Use is subject to license terms + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "myisamdef.h" + + /* Read first row through a specific key */ + +int mi_rfirst(MI_INFO *info, uchar *buf, int inx) +{ + DBUG_ENTER("mi_rfirst"); + info->lastpos= HA_OFFSET_ERROR; + info->update|= HA_STATE_PREV_FOUND; + DBUG_RETURN(mi_rnext(info,buf,inx)); +} /* mi_rfirst */ diff --git a/storage/myisam/mi_rkey.c b/storage/myisam/mi_rkey.c new file mode 100644 index 00000000..bf6f3ef8 --- /dev/null +++ b/storage/myisam/mi_rkey.c @@ -0,0 +1,265 @@ +/* Copyright (c) 2000-2007 MySQL AB, 2009 Sun Microsystems, Inc. + Use is subject to license terms. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Read record based on a key */ + +#include "myisamdef.h" +#include "rt_index.h" + + /* Read a record using key */ + /* Ordinary search_flag is 0 ; Give error if no record with key */ + +int mi_rkey(MI_INFO *info, uchar *buf, int inx, const uchar *key, + key_part_map keypart_map, enum ha_rkey_function search_flag) +{ + uchar *key_buff; + MYISAM_SHARE *share=info->s; + MI_KEYDEF *keyinfo; + HA_KEYSEG *last_used_keyseg; + uint pack_key_length, use_key_length, nextflag; + check_result_t res= CHECK_NEG; + DBUG_ENTER("mi_rkey"); + DBUG_PRINT("enter", ("base: %p buf: %p inx: %d search_flag: %d", + info, buf, inx, search_flag)); + + if ((inx = _mi_check_index(info,inx)) < 0) + DBUG_RETURN(my_errno); + + info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + info->last_key_func= search_flag; + keyinfo= share->keyinfo + inx; + + if (info->once_flags & USE_PACKED_KEYS) + { + info->once_flags&= ~USE_PACKED_KEYS; /* Reset flag */ + /* + key is already packed!; This happens when we are using a MERGE TABLE + In this key 'key_part_map' is the length of the key ! + */ + key_buff=info->lastkey+info->s->base.max_key_length; + pack_key_length= keypart_map; + bmove(key_buff, key, pack_key_length); + last_used_keyseg= info->s->keyinfo[inx].seg + info->last_used_keyseg; + } + else + { + DBUG_ASSERT(keypart_map); + /* Save the packed key for later use in the second buffer of lastkey. */ + key_buff=info->lastkey+info->s->base.max_key_length; + pack_key_length=_mi_pack_key(info,(uint) inx, key_buff, (uchar*) key, + keypart_map, &last_used_keyseg); + /* Save packed_key_length for use by the MERGE engine. */ + info->pack_key_length= pack_key_length; + info->last_used_keyseg= (uint16) (last_used_keyseg - + info->s->keyinfo[inx].seg); + DBUG_EXECUTE("key",_mi_print_key(DBUG_FILE, keyinfo->seg, + key_buff, pack_key_length);); + } + + if (fast_mi_readinfo(info)) + goto err; + + if (share->concurrent_insert) + mysql_rwlock_rdlock(&share->key_root_lock[inx]); + + nextflag=myisam_read_vec[search_flag]; + use_key_length=pack_key_length; + if (!(nextflag & (SEARCH_FIND | SEARCH_NO_FIND | SEARCH_LAST))) + use_key_length=USE_WHOLE_KEY; + + switch (info->s->keyinfo[inx].key_alg) { +#ifdef HAVE_RTREE_KEYS + case HA_KEY_ALG_RTREE: + if (rtree_find_first(info,inx,key_buff,use_key_length,nextflag) < 0) + { + mi_print_error(info->s, HA_ERR_CRASHED); + my_errno=HA_ERR_CRASHED; + if (share->concurrent_insert) + mysql_rwlock_unlock(&share->key_root_lock[inx]); + fast_mi_writeinfo(info); + goto err; + } + break; +#endif + case HA_KEY_ALG_BTREE: + default: + if (!_mi_search(info, keyinfo, key_buff, use_key_length, + myisam_read_vec[search_flag], info->s->state.key_root[inx])) + { + /* + Found a key, but it might not be usable. We cannot use rows that + are inserted by other threads after we got our table lock + ("concurrent inserts"). The record may not even be present yet. + Keys are inserted into the index(es) before the record is + inserted into the data file. When we got our table lock, we + saved the current data_file_length. Concurrent inserts always go + to the end of the file. So we can test if the found key + references a new record. + + If we are searching for a partial key (or using >, >=, < or <=) and + the data is outside of the data file, we need to continue searching + for the first key inside the data file. + + We do also continue searching if an index condition check function + is available. + */ + while ((info->lastpos >= info->state->data_file_length && + (search_flag != HA_READ_KEY_EXACT || + last_used_keyseg != keyinfo->seg + keyinfo->keysegs)) || + (res= mi_check_index_tuple(info, inx, buf)) == CHECK_NEG) + { + uint not_used[2]; + /* + Skip rows that are inserted by other threads since we got a lock + Note that this can only happen if we are not searching after an + full length exact key, because the keys are sorted + according to position + */ + if (_mi_search_next(info, keyinfo, info->lastkey, + info->lastkey_length, + myisam_readnext_vec[search_flag], + info->s->state.key_root[inx])) + { + info->lastpos= HA_OFFSET_ERROR; + break; + } + /* + Check that the found key does still match the search. + _mi_search_next() delivers the next key regardless of its + value. + */ + if (search_flag == HA_READ_KEY_EXACT && + ha_key_cmp(keyinfo->seg, key_buff, info->lastkey, use_key_length, + SEARCH_FIND, not_used)) + { + my_errno= HA_ERR_KEY_NOT_FOUND; + info->lastpos= HA_OFFSET_ERROR; + break; + } + /* + If we are at the last key on the key page, allow writers to + access the index. + */ + if (info->int_keypos >= info->int_maxpos && + mi_yield_and_check_if_killed(info, inx)) + { + /* Aborted by user */ + DBUG_ASSERT(info->lastpos == HA_OFFSET_ERROR && + my_errno == HA_ERR_ABORTED_BY_USER); + res= CHECK_ERROR; + buf= 0; /* Fast abort */ + break; + } + } + if (res == CHECK_OUT_OF_RANGE) + { + /* Change error from HA_ERR_END_OF_FILE */ + DBUG_ASSERT(info->lastpos == HA_OFFSET_ERROR); + my_errno= HA_ERR_KEY_NOT_FOUND; + } + /* + Error if no row found within the data file. (Bug #29838) + Do not overwrite my_errno if already at HA_OFFSET_ERROR. + */ + if (info->lastpos != HA_OFFSET_ERROR && + info->lastpos >= info->state->data_file_length) + { + info->lastpos= HA_OFFSET_ERROR; + my_errno= HA_ERR_KEY_NOT_FOUND; + } + } + else + { + DBUG_ASSERT(info->lastpos == HA_OFFSET_ERROR); + } + } + if (share->concurrent_insert) + mysql_rwlock_unlock(&share->key_root_lock[inx]); + + info->last_rkey_length= pack_key_length; + + if (info->lastpos == HA_OFFSET_ERROR) /* No such record */ + { + fast_mi_writeinfo(info); + if (!buf) + DBUG_RETURN(my_errno); + } + else + { + /* Calculate length of the found key; Used by mi_rnext_same */ + if ((keyinfo->flag & HA_VAR_LENGTH_KEY) && last_used_keyseg) + info->last_rkey_length= _mi_keylength_part(keyinfo, info->lastkey, + last_used_keyseg); + + /* Check if we don't want to have record back, only error message */ + if (!buf) + { + fast_mi_writeinfo(info); + DBUG_RETURN(0); + } + if (!(*info->read_record)(info,info->lastpos,buf)) + { + info->update|= HA_STATE_AKTIV; /* Record is read */ + DBUG_RETURN(0); + } + DBUG_PRINT("error", ("Didn't find row. Error %d", my_errno)); + info->lastpos= HA_OFFSET_ERROR; /* Didn't find row */ + } + + /* Store last used key as a base for read next */ + memcpy(info->lastkey,key_buff,pack_key_length); + info->last_rkey_length= pack_key_length; + bzero((char*) info->lastkey+pack_key_length,info->s->base.rec_reflength); + info->lastkey_length=pack_key_length+info->s->base.rec_reflength; + + if (search_flag == HA_READ_AFTER_KEY) + info->update|=HA_STATE_NEXT_FOUND; /* Previous gives last row */ +err: + DBUG_RETURN(my_errno); +} /* _mi_rkey */ + + +/* + Yield to possible other writers during a index scan. + Check also if we got killed by the user and if yes, return + HA_ERR_LOCK_WAIT_TIMEOUT + + return 0 ok + return 1 Query has been requested to be killed +*/ + +my_bool mi_yield_and_check_if_killed(MI_INFO *info, int inx) +{ + MYISAM_SHARE *share; + if (mi_killed(info)) + { + /* purecov: begin tested */ + info->lastpos= HA_OFFSET_ERROR; + /* Set error that we where aborted by kill from application */ + my_errno= HA_ERR_ABORTED_BY_USER; + return 1; + /* purecov: end */ + + } + + if ((share= info->s)->concurrent_insert) + { + /* Give writers a chance to access index */ + mysql_rwlock_unlock(&share->key_root_lock[inx]); + mysql_rwlock_rdlock(&share->key_root_lock[inx]); + } + return 0; +} diff --git a/storage/myisam/mi_rlast.c b/storage/myisam/mi_rlast.c new file mode 100644 index 00000000..30cd17d9 --- /dev/null +++ b/storage/myisam/mi_rlast.c @@ -0,0 +1,27 @@ +/* Copyright (c) 2000, 2001, 2005-2007 MySQL AB + Use is subject to license terms + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "myisamdef.h" + + /* Read last row with the same key as the previous read. */ + +int mi_rlast(MI_INFO *info, uchar *buf, int inx) +{ + DBUG_ENTER("mi_rlast"); + info->lastpos= HA_OFFSET_ERROR; + info->update|= HA_STATE_NEXT_FOUND; + DBUG_RETURN(mi_rprev(info,buf,inx)); +} /* mi_rlast */ diff --git a/storage/myisam/mi_rnext.c b/storage/myisam/mi_rnext.c new file mode 100644 index 00000000..7124c102 --- /dev/null +++ b/storage/myisam/mi_rnext.c @@ -0,0 +1,155 @@ +/* + Copyright (c) 2000, 2010, Oracle and/or its affiliates + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "myisamdef.h" +#include "rt_index.h" + + /* + Read next row with the same key as previous read + One may have done a write, update or delete of the previous row. + NOTE! Even if one changes the previous row, the next read is done + based on the position of the last used key! + */ + +int mi_rnext(MI_INFO *info, uchar *buf, int inx) +{ + int error,changed; + uint flag; + check_result_t check= CHECK_POS; + uint update_mask= HA_STATE_NEXT_FOUND; + DBUG_ENTER("mi_rnext"); + + if ((inx = _mi_check_index(info,inx)) < 0) + DBUG_RETURN(my_errno); + flag=SEARCH_BIGGER; /* Read next */ + if (info->lastpos == HA_OFFSET_ERROR && info->update & HA_STATE_PREV_FOUND) + flag=0; /* Read first */ + + if (fast_mi_readinfo(info)) + DBUG_RETURN(my_errno); + if (info->s->concurrent_insert) + mysql_rwlock_rdlock(&info->s->key_root_lock[inx]); + changed=_mi_test_if_changed(info); + if (!flag) + { + switch(info->s->keyinfo[inx].key_alg){ +#ifdef HAVE_RTREE_KEYS + case HA_KEY_ALG_RTREE: + error=rtree_get_first(info,inx,info->lastkey_length); + break; +#endif + case HA_KEY_ALG_BTREE: + default: + error=_mi_search_first(info,info->s->keyinfo+inx, + info->s->state.key_root[inx]); + break; + } + /* + "search first" failed. This means we have no pivot for + "search next", or in other words MI_INFO::lastkey is + likely uninitialized. + + Normally SQL layer would never request "search next" if + "search first" failed. But HANDLER may do anything. + + As mi_rnext() without preceding mi_rkey()/mi_rfirst() + equals to mi_rfirst(), we must restore original state + as if failing mi_rfirst() was not called. + */ + if (error) + update_mask|= HA_STATE_PREV_FOUND; + } + else + { + switch (info->s->keyinfo[inx].key_alg) { +#ifdef HAVE_RTREE_KEYS + case HA_KEY_ALG_RTREE: + /* + Note that rtree doesn't support that the table + may be changed since last call, so we do need + to skip rows inserted by other threads like in btree + */ + error= rtree_get_next(info,inx,info->lastkey_length); + break; +#endif + case HA_KEY_ALG_BTREE: + default: + if (!changed) + error= _mi_search_next(info,info->s->keyinfo+inx,info->lastkey, + info->lastkey_length,flag, + info->s->state.key_root[inx]); + else + error= _mi_search(info,info->s->keyinfo+inx,info->lastkey, + USE_WHOLE_KEY,flag, info->s->state.key_root[inx]); + } + } + + if (!error) + { + while ((info->s->concurrent_insert && + info->lastpos >= info->state->data_file_length) || + (check= mi_check_index_tuple(info, inx, buf)) == CHECK_NEG) + { + /* + If we are at the last key on the key page, allow writers to + access the index. + */ + if (info->int_keypos >= info->int_maxpos && + mi_yield_and_check_if_killed(info, inx)) + { + error= 1; + break; + } + + /* + Skip rows that are either inserted by other threads since + we got a lock or do not match pushed index conditions + */ + if ((error=_mi_search_next(info,info->s->keyinfo+inx, + info->lastkey, + info->lastkey_length, + SEARCH_BIGGER, + info->s->state.key_root[inx]))) + break; + } + } + + if (info->s->concurrent_insert) + mysql_rwlock_unlock(&info->s->key_root_lock[inx]); + + /* Don't clear if database-changed */ + info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + info->update|= update_mask; + + if (error || check != CHECK_POS) + { + fast_mi_writeinfo(info); + if (my_errno == HA_ERR_KEY_NOT_FOUND) + my_errno=HA_ERR_END_OF_FILE; + } + else if (!buf) + { + fast_mi_writeinfo(info); + DBUG_RETURN(info->lastpos==HA_OFFSET_ERROR ? my_errno : 0); + } + else if (!(*info->read_record)(info,info->lastpos,buf)) + { + info->update|= HA_STATE_AKTIV; /* Record is read */ + DBUG_RETURN(0); + } + DBUG_PRINT("error",("Got error: %d, errno: %d",error, my_errno)); + DBUG_RETURN(my_errno); +} /* mi_rnext */ diff --git a/storage/myisam/mi_rnext_same.c b/storage/myisam/mi_rnext_same.c new file mode 100644 index 00000000..480b54f7 --- /dev/null +++ b/storage/myisam/mi_rnext_same.c @@ -0,0 +1,126 @@ +/* Copyright (c) 2000-2007 MySQL AB, 2009 Sun Microsystems, Inc. + Use is subject to license terms. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "myisamdef.h" +#include "rt_index.h" + + /* + Read next row with the same key as previous read, but abort if + the key changes. + One may have done a write, update or delete of the previous row. + NOTE! Even if one changes the previous row, the next read is done + based on the position of the last used key! + */ + +int mi_rnext_same(MI_INFO *info, uchar *buf) +{ + int error; + uint inx,not_used[2]; + MI_KEYDEF *keyinfo; + check_result_t check= CHECK_POS; + DBUG_ENTER("mi_rnext_same"); + + if ((int) (inx=info->lastinx) < 0 || info->lastpos == HA_OFFSET_ERROR) + DBUG_RETURN(my_errno=HA_ERR_WRONG_INDEX); + keyinfo=info->s->keyinfo+inx; + if (fast_mi_readinfo(info)) + DBUG_RETURN(my_errno); + + if (info->s->concurrent_insert) + mysql_rwlock_rdlock(&info->s->key_root_lock[inx]); + + switch (keyinfo->key_alg) + { +#ifdef HAVE_RTREE_KEYS + case HA_KEY_ALG_RTREE: + if ((error=rtree_find_next(info,inx, + myisam_read_vec[info->last_key_func]))) + { + error=1; + my_errno=HA_ERR_END_OF_FILE; + info->lastpos= HA_OFFSET_ERROR; + break; + } + break; +#endif + case HA_KEY_ALG_BTREE: + default: + if (!(info->update & HA_STATE_RNEXT_SAME)) + { + /* First rnext_same; Store old key */ + memcpy(info->lastkey2,info->lastkey,info->last_rkey_length); + } + for (;;) + { + /* + If we are at the last key on the key page, allow writers to + access the index. + */ + if (info->int_keypos >= info->int_maxpos && + mi_yield_and_check_if_killed(info, inx)) + { + error=1; + break; + } + + if ((error=_mi_search_next(info,keyinfo,info->lastkey, + info->lastkey_length,SEARCH_BIGGER, + info->s->state.key_root[inx]))) + break; + if (ha_key_cmp(keyinfo->seg, info->lastkey, info->lastkey2, + info->last_rkey_length, SEARCH_FIND, not_used)) + { + error=1; + my_errno=HA_ERR_END_OF_FILE; + info->lastpos= HA_OFFSET_ERROR; + break; + } + /* + Skip + - rows that are inserted by other threads since we got a lock + - rows that don't match index condition + */ + if (info->lastpos < info->state->data_file_length && + (check= mi_check_index_tuple(info, inx, buf)) != CHECK_NEG) + break; + } + } + if (info->s->concurrent_insert) + mysql_rwlock_unlock(&info->s->key_root_lock[inx]); + + + /* Don't clear if database-changed */ + info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + info->update|= HA_STATE_NEXT_FOUND | HA_STATE_RNEXT_SAME; + + if (error || check != CHECK_POS) + { + fast_mi_writeinfo(info); + if (my_errno == HA_ERR_KEY_NOT_FOUND) + my_errno=HA_ERR_END_OF_FILE; + } + else if (!buf) + { + fast_mi_writeinfo(info); + DBUG_RETURN(info->lastpos==HA_OFFSET_ERROR ? my_errno : 0); + } + else if (!(*info->read_record)(info,info->lastpos,buf)) + { + info->update|= HA_STATE_AKTIV; /* Record is read */ + DBUG_RETURN(0); + } + DBUG_RETURN(my_errno); +} /* mi_rnext_same */ diff --git a/storage/myisam/mi_rprev.c b/storage/myisam/mi_rprev.c new file mode 100644 index 00000000..698e3b59 --- /dev/null +++ b/storage/myisam/mi_rprev.c @@ -0,0 +1,112 @@ +/* Copyright (c) 2000, 2001, 2004-2007 MySQL AB, 2009 Sun Microsystems, Inc. + Use is subject to license terms. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "myisamdef.h" + + /* + Read previous row with the same key as previous read + One may have done a write, update or delete of the previous row. + NOTE! Even if one changes the previous row, the next read is done + based on the position of the last used key! + */ + +int mi_rprev(MI_INFO *info, uchar *buf, int inx) +{ + int error,changed; + register uint flag; + MYISAM_SHARE *share=info->s; + check_result_t check= CHECK_POS; + DBUG_ENTER("mi_rprev"); + + if ((inx = _mi_check_index(info,inx)) < 0) + DBUG_RETURN(my_errno); + flag=SEARCH_SMALLER; /* Read previous */ + if (info->lastpos == HA_OFFSET_ERROR && info->update & HA_STATE_NEXT_FOUND) + flag=0; /* Read last */ + + if (fast_mi_readinfo(info)) + DBUG_RETURN(my_errno); + changed=_mi_test_if_changed(info); + if (share->concurrent_insert) + mysql_rwlock_rdlock(&share->key_root_lock[inx]); + if (!flag) + error=_mi_search_last(info, share->keyinfo+inx, + share->state.key_root[inx]); + else if (!changed) + error=_mi_search_next(info,share->keyinfo+inx,info->lastkey, + info->lastkey_length,flag, + share->state.key_root[inx]); + else + error=_mi_search(info,share->keyinfo+inx,info->lastkey, + USE_WHOLE_KEY, flag, share->state.key_root[inx]); + + if (!error) + { + my_off_t cur_keypage= info->last_keypage; + while ((share->concurrent_insert && + info->lastpos >= info->state->data_file_length) || + (check= mi_check_index_tuple(info, inx, buf)) == CHECK_NEG) + { + /* + If we are at the last (i.e. first?) key on the key page, + allow writers to access the index. + */ + if (info->last_keypage != cur_keypage) + { + cur_keypage= info->last_keypage; + if (mi_yield_and_check_if_killed(info, inx)) + { + error= 1; + break; + } + } + + /* + Skip rows that are either inserted by other threads since + we got a lock or do not match pushed index conditions + */ + if ((error=_mi_search_next(info,share->keyinfo+inx,info->lastkey, + info->lastkey_length, + SEARCH_SMALLER, + share->state.key_root[inx]))) + break; + } + } + + if (share->concurrent_insert) + mysql_rwlock_unlock(&share->key_root_lock[inx]); + + info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + info->update|= HA_STATE_PREV_FOUND; + + if (error || check != CHECK_POS) + { + fast_mi_writeinfo(info); + if (my_errno == HA_ERR_KEY_NOT_FOUND) + my_errno=HA_ERR_END_OF_FILE; + } + else if (!buf) + { + fast_mi_writeinfo(info); + DBUG_RETURN(info->lastpos==HA_OFFSET_ERROR ? my_errno : 0); + } + else if (!(*info->read_record)(info,info->lastpos,buf)) + { + info->update|= HA_STATE_AKTIV; /* Record is read */ + DBUG_RETURN(0); + } + DBUG_RETURN(my_errno); +} /* mi_rprev */ diff --git a/storage/myisam/mi_rrnd.c b/storage/myisam/mi_rrnd.c new file mode 100644 index 00000000..4e49e792 --- /dev/null +++ b/storage/myisam/mi_rrnd.c @@ -0,0 +1,60 @@ +/* Copyright (c) 2000-2002, 2004-2007 MySQL AB + Use is subject to license terms + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Read a record with random-access. The position to the record must + get by MI_INFO. The next record can be read with pos= MI_POS_ERROR */ + + +#include "myisamdef.h" + +/* + Read a row based on position. + If filepos= HA_OFFSET_ERROR then read next row + Return values + Returns one of following values: + 0 = Ok. + HA_ERR_RECORD_DELETED = Record is deleted. + HA_ERR_END_OF_FILE = EOF. +*/ + +int mi_rrnd(MI_INFO *info, uchar *buf, register my_off_t filepos) +{ + my_bool skip_deleted_blocks; + DBUG_ENTER("mi_rrnd"); + + skip_deleted_blocks=0; + + if (filepos == HA_OFFSET_ERROR) + { + skip_deleted_blocks=1; + if (info->lastpos == HA_OFFSET_ERROR) /* First read ? */ + filepos= info->s->pack.header_length; /* Read first record */ + else + filepos= info->nextpos; + } + + if (info->once_flags & RRND_PRESERVE_LASTINX) + info->once_flags&= ~RRND_PRESERVE_LASTINX; + else + info->lastinx= -1; /* Can't forward or backward */ + /* Init all but update-flag */ + info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + + if (info->opt_flag & WRITE_CACHE_USED && flush_io_cache(&info->rec_cache)) + DBUG_RETURN(my_errno); + + DBUG_RETURN ((*info->s->read_rnd)(info,buf,filepos,skip_deleted_blocks)); +} diff --git a/storage/myisam/mi_rsame.c b/storage/myisam/mi_rsame.c new file mode 100644 index 00000000..7511531b --- /dev/null +++ b/storage/myisam/mi_rsame.c @@ -0,0 +1,66 @@ +/* Copyright (c) 2000, 2001, 2005-2007 MySQL AB, 2009 Sun Microsystems, Inc. + Use is subject to license terms. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "myisamdef.h" + + /* + ** Find current row with read on position or read on key + ** If inx >= 0 find record using key + ** Return values: + ** 0 = Ok. + ** HA_ERR_KEY_NOT_FOUND = Row is deleted + ** HA_ERR_END_OF_FILE = End of file + */ + + +int mi_rsame(MI_INFO *info, uchar *record, int inx) +{ + DBUG_ENTER("mi_rsame"); + + if (inx != -1 && ! mi_is_key_active(info->s->state.key_map, inx)) + { + DBUG_RETURN(my_errno=HA_ERR_WRONG_INDEX); + } + if (info->lastpos == HA_OFFSET_ERROR || info->update & HA_STATE_DELETED) + { + DBUG_RETURN(my_errno=HA_ERR_KEY_NOT_FOUND); /* No current record */ + } + info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + + /* Read row from data file */ + if (fast_mi_readinfo(info)) + DBUG_RETURN(my_errno); + + if (inx >= 0) + { + info->lastinx=inx; + info->lastkey_length=_mi_make_key(info,(uint) inx,info->lastkey,record, + info->lastpos); + if (info->s->concurrent_insert) + mysql_rwlock_rdlock(&info->s->key_root_lock[inx]); + (void) _mi_search(info,info->s->keyinfo+inx,info->lastkey, USE_WHOLE_KEY, + SEARCH_SAME, + info->s->state.key_root[inx]); + if (info->s->concurrent_insert) + mysql_rwlock_unlock(&info->s->key_root_lock[inx]); + } + + if (!(*info->read_record)(info,info->lastpos,record)) + DBUG_RETURN(0); + if (my_errno == HA_ERR_RECORD_DELETED) + my_errno=HA_ERR_KEY_NOT_FOUND; + DBUG_RETURN(my_errno); +} /* mi_rsame */ diff --git a/storage/myisam/mi_rsamepos.c b/storage/myisam/mi_rsamepos.c new file mode 100644 index 00000000..4570aa3b --- /dev/null +++ b/storage/myisam/mi_rsamepos.c @@ -0,0 +1,58 @@ +/* Copyright (c) 2000, 2001, 2005-2007 MySQL AB + Use is subject to license terms + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* read record through position and fix key-position */ +/* As mi_rsame but supply a position */ + +#include "myisamdef.h" + + + /* + ** If inx >= 0 update index pointer + ** Returns one of the following values: + ** 0 = Ok. + ** HA_ERR_KEY_NOT_FOUND = Row is deleted + ** HA_ERR_END_OF_FILE = End of file + */ + +int mi_rsame_with_pos(MI_INFO *info, uchar *record, int inx, my_off_t filepos) +{ + DBUG_ENTER("mi_rsame_with_pos"); + DBUG_PRINT("enter",("index: %d filepos: %ld", inx, (long) filepos)); + + if (inx < -1 || + (inx >= 0 && ! mi_is_key_active(info->s->state.key_map, inx))) + { + DBUG_RETURN(my_errno=HA_ERR_WRONG_INDEX); + } + + info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + if ((*info->s->read_rnd)(info,record,filepos,0)) + { + if (my_errno == HA_ERR_RECORD_DELETED) + my_errno=HA_ERR_KEY_NOT_FOUND; + DBUG_RETURN(my_errno); + } + info->lastpos=filepos; + info->lastinx=inx; + if (inx >= 0) + { + info->lastkey_length=_mi_make_key(info,(uint) inx,info->lastkey,record, + info->lastpos); + info->update|=HA_STATE_KEY_CHANGED; /* Don't use indexposition */ + } + DBUG_RETURN(0); +} /* mi_rsame_pos */ diff --git a/storage/myisam/mi_scan.c b/storage/myisam/mi_scan.c new file mode 100644 index 00000000..8d436c4e --- /dev/null +++ b/storage/myisam/mi_scan.c @@ -0,0 +1,46 @@ +/* Copyright (c) 2000, 2001, 2005-2007 MySQL AB + Use is subject to license terms + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Read through all rows sequntially */ + +#include "myisamdef.h" + +int mi_scan_init(register MI_INFO *info) +{ + DBUG_ENTER("mi_scan_init"); + info->nextpos=info->s->pack.header_length; /* Read first record */ + info->lastinx= -1; /* Can't forward or backward */ + if (info->opt_flag & WRITE_CACHE_USED && flush_io_cache(&info->rec_cache)) + DBUG_RETURN(my_errno); + DBUG_RETURN(0); +} + +/* + Read a row based on position. + If filepos= HA_OFFSET_ERROR then read next row + Return values + Returns one of following values: + 0 = Ok. + HA_ERR_END_OF_FILE = EOF. +*/ + +int mi_scan(MI_INFO *info, uchar *buf) +{ + DBUG_ENTER("mi_scan"); + /* Init all but update-flag */ + info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + DBUG_RETURN ((*info->s->read_rnd)(info,buf,info->nextpos,1)); +} diff --git a/storage/myisam/mi_search.c b/storage/myisam/mi_search.c new file mode 100644 index 00000000..dbad6aa7 --- /dev/null +++ b/storage/myisam/mi_search.c @@ -0,0 +1,1918 @@ +/* + Copyright (c) 2000, 2010, Oracle and/or its affiliates + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* key handling functions */ + +#include "fulltext.h" +#include "m_ctype.h" + +static my_bool _mi_get_prev_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, + uchar *key, uchar *keypos, + uint *return_key_length); + + /* Check index */ + +int _mi_check_index(MI_INFO *info, int inx) +{ + if (inx == -1) /* Use last index */ + inx=info->lastinx; + if (inx < 0) + { + my_errno= HA_ERR_WRONG_INDEX; + return -1; + } + if (!mi_is_key_active(info->s->state.key_map, inx)) + { + my_errno= info->s->state.state.records ? HA_ERR_WRONG_INDEX : + HA_ERR_END_OF_FILE; + return -1; + } + if (info->lastinx != inx) /* Index changed */ + { + info->lastinx = inx; + info->page_changed=1; + info->update= ((info->update & (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED)) | + HA_STATE_NEXT_FOUND | HA_STATE_PREV_FOUND); + } + if (info->opt_flag & WRITE_CACHE_USED && flush_io_cache(&info->rec_cache)) + return(-1); + return(inx); +} /* mi_check_index */ + + + /* + ** Search after row by a key + ** Position to row is stored in info->lastpos + ** Return: -1 if not found + ** 1 if one should continue search on higher level + */ + +int _mi_search(register MI_INFO *info, register MI_KEYDEF *keyinfo, + uchar *key, uint key_len, uint nextflag, register my_off_t pos) +{ + my_bool last_key; + int error,flag; + uint nod_flag; + uchar *keypos,*maxpos; + uchar lastkey[HA_MAX_KEY_BUFF],*buff; + DBUG_ENTER("_mi_search"); + DBUG_PRINT("enter",("pos: %lu nextflag: %u lastpos: %lu", + (ulong) pos, nextflag, (ulong) info->lastpos)); + DBUG_EXECUTE("key",_mi_print_key(DBUG_FILE,keyinfo->seg,key,key_len);); + + if (pos == HA_OFFSET_ERROR) + { + my_errno=HA_ERR_KEY_NOT_FOUND; /* Didn't find key */ + info->lastpos= HA_OFFSET_ERROR; + if (!(nextflag & (SEARCH_SMALLER | SEARCH_BIGGER | SEARCH_LAST))) + DBUG_RETURN(-1); /* Not found ; return error */ + DBUG_RETURN(1); /* Search at upper levels */ + } + + if (!(buff=_mi_fetch_keypage(info,keyinfo,pos,DFLT_INIT_HITS,info->buff, + MY_TEST(!(nextflag & SEARCH_SAVE_BUFF))))) + goto err; + DBUG_DUMP("page", buff, mi_getint(buff)); + + flag=(*keyinfo->bin_search)(info,keyinfo,buff,key,key_len,nextflag, + &keypos,lastkey, &last_key); + if (flag == MI_FOUND_WRONG_KEY) + { + my_errno= HA_ERR_CRASHED; + goto err; + } + nod_flag=mi_test_if_nod(buff); + maxpos=buff+mi_getint(buff)-1; + + if (flag) + { + if ((error=_mi_search(info,keyinfo,key,key_len,nextflag, + _mi_kpos(nod_flag,keypos))) <= 0) + DBUG_RETURN(error); + + if (flag >0) + { + if (nextflag & (SEARCH_SMALLER | SEARCH_LAST) && + keypos == buff+2+nod_flag) + DBUG_RETURN(1); /* Bigger than key */ + } + else if (nextflag & SEARCH_BIGGER && keypos >= maxpos) + DBUG_RETURN(1); /* Smaller than key */ + } + else + { + if ((nextflag & SEARCH_FIND) && nod_flag && + ((keyinfo->flag & (HA_NOSAME | HA_NULL_PART)) != HA_NOSAME || + key_len != USE_WHOLE_KEY)) + { + if ((error=_mi_search(info,keyinfo,key,key_len,SEARCH_FIND, + _mi_kpos(nod_flag,keypos))) >= 0 || + my_errno != HA_ERR_KEY_NOT_FOUND) + DBUG_RETURN(error); + info->last_keypage= HA_OFFSET_ERROR; /* Buffer not in mem */ + } + } + if (pos != info->last_keypage) + { + uchar *old_buff=buff; + if (!(buff=_mi_fetch_keypage(info,keyinfo,pos,DFLT_INIT_HITS,info->buff, + MY_TEST(!(nextflag & SEARCH_SAVE_BUFF))))) + goto err; + keypos=buff+(keypos-old_buff); + maxpos=buff+(maxpos-old_buff); + } + + if ((nextflag & (SEARCH_SMALLER | SEARCH_LAST)) && flag != 0) + { + uint not_used[2]; + if (_mi_get_prev_key(info,keyinfo, buff, info->lastkey, keypos, + &info->lastkey_length)) + goto err; + if (!(nextflag & SEARCH_SMALLER) && + ha_key_cmp(keyinfo->seg, info->lastkey, key, key_len, SEARCH_FIND, + not_used)) + { + my_errno=HA_ERR_KEY_NOT_FOUND; /* Didn't find key */ + goto err; + } + } + else + { + info->lastkey_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,lastkey); + if (!info->lastkey_length) + goto err; + memcpy(info->lastkey,lastkey,info->lastkey_length); + } + info->lastpos=_mi_dpos(info,0,info->lastkey+info->lastkey_length); + /* Save position for a possible read next / previous */ + info->int_keypos=info->buff+ (keypos-buff); + info->int_maxpos=info->buff+ (maxpos-buff); + info->int_nod_flag=nod_flag; + info->int_keytree_version=keyinfo->version; + info->last_search_keypage=info->last_keypage; + info->page_changed=0; + info->buff_used= (info->buff != buff); /* If we have to reread buff */ + + DBUG_PRINT("exit",("found key at %lu",(ulong) info->lastpos)); + DBUG_RETURN(0); + +err: + DBUG_PRINT("exit",("Error: %d",my_errno)); + info->lastpos= HA_OFFSET_ERROR; + info->page_changed=1; + DBUG_RETURN (-1); +} /* _mi_search */ + + + /* Search after key in page-block */ + /* If packed key puts smaller or identical key in buff */ + /* ret_pos point to where find or bigger key starts */ + /* ARGSUSED */ + +int _mi_bin_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page, + uchar *key, uint key_len, uint comp_flag, uchar **ret_pos, + uchar *buff __attribute__((unused)), my_bool *last_key) +{ + reg4 int start,mid,end,save_end; + int UNINIT_VAR(flag); + uint totlength,nod_flag,not_used[2]; + DBUG_ENTER("_mi_bin_search"); + + totlength=keyinfo->keylength+(nod_flag=mi_test_if_nod(page)); + start=0; mid=1; + save_end=end=(int) ((mi_getint(page)-2-nod_flag)/totlength-1); + DBUG_PRINT("test",("mi_getint: %d end: %d",mi_getint(page),end)); + page+=2+nod_flag; + + while (start != end) + { + mid= (start+end)/2; + if ((flag=ha_key_cmp(keyinfo->seg,page+(uint) mid*totlength,key,key_len, + comp_flag, not_used)) + >= 0) + end=mid; + else + start=mid+1; + } + if (mid != start) + flag=ha_key_cmp(keyinfo->seg,page+(uint) start*totlength,key,key_len, + comp_flag, not_used); + if (flag < 0) + start++; /* point at next, bigger key */ + *ret_pos=page+(uint) start*totlength; + *last_key= end == save_end; + DBUG_PRINT("exit",("flag: %d keypos: %d",flag,start)); + DBUG_RETURN(flag); +} /* _mi_bin_search */ + + +/* + Locate a packed key in a key page. + + SYNOPSIS + _mi_seq_search() + info Open table information. + keyinfo Key definition information. + page Key page (beginning). + key Search key. + key_len Length to use from search key or USE_WHOLE_KEY + comp_flag Search flags like SEARCH_SAME etc. + ret_pos RETURN Position in key page behind this key. + buff RETURN Copy of previous or identical unpacked key. + last_key RETURN If key is last in page. + + DESCRIPTION + Used instead of _mi_bin_search() when key is packed. + Puts smaller or identical key in buff. + Key is searched sequentially. + + RETURN + > 0 Key in 'buff' is smaller than search key. + 0 Key in 'buff' is identical to search key. + < 0 Not found. +*/ + +int _mi_seq_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page, + uchar *key, uint key_len, uint comp_flag, uchar **ret_pos, + uchar *buff, my_bool *last_key) +{ + int UNINIT_VAR(flag); + uint nod_flag,UNINIT_VAR(length),not_used[2]; + uchar t_buff[HA_MAX_KEY_BUFF],*end; + DBUG_ENTER("_mi_seq_search"); + + end= page+mi_getint(page); + nod_flag=mi_test_if_nod(page); + page+=2+nod_flag; + *ret_pos=page; + t_buff[0]=0; /* Avoid bugs */ + while (page < end) + { + length=(*keyinfo->get_key)(keyinfo,nod_flag,&page,t_buff); + if (length == 0 || page > end) + { + mi_print_error(info->s, HA_ERR_CRASHED); + my_errno=HA_ERR_CRASHED; + DBUG_PRINT("error", + ("Found wrong key: length: %u page: %p end: %p", + length, page, end)); + DBUG_RETURN(MI_FOUND_WRONG_KEY); + } + if ((flag=ha_key_cmp(keyinfo->seg,t_buff,key,key_len,comp_flag, + not_used)) >= 0) + break; +#ifdef EXTRA_DEBUG + DBUG_PRINT("loop",("page: 0x%lx key: '%s' flag: %d", (long) page, t_buff, + flag)); +#endif + memcpy(buff,t_buff,length); + *ret_pos=page; + } + if (flag == 0) + memcpy(buff,t_buff,length); /* Result is first key */ + *last_key= page == end; + DBUG_PRINT("exit",("flag: %d ret_pos: %p", flag, *ret_pos)); + DBUG_RETURN(flag); +} /* _mi_seq_search */ + + +int _mi_prefix_search(MI_INFO *info, register MI_KEYDEF *keyinfo, uchar *page, + uchar *key, uint key_len, uint nextflag, uchar **ret_pos, + uchar *buff, my_bool *last_key) +{ + /* + my_flag is raw comparison result to be changed according to + SEARCH_NO_FIND,SEARCH_LAST and HA_REVERSE_SORT flags. + flag is the value returned by ha_key_cmp and as treated as final + */ + int flag=0, my_flag=-1; + uint nod_flag, UNINIT_VAR(length), len, matched, cmplen, kseg_len; + uint UNINIT_VAR(prefix_len), suffix_len; + int key_len_skip, UNINIT_VAR(seg_len_pack), key_len_left; + uchar *end, *kseg, *vseg; + const uchar *sort_order= keyinfo->seg->charset->sort_order; + uchar tt_buff[HA_MAX_KEY_BUFF+2], *t_buff=tt_buff+2; + uchar *UNINIT_VAR(saved_from), *UNINIT_VAR(saved_to); + uchar *UNINIT_VAR(saved_vseg); + uint saved_length=0, saved_prefix_len=0; + uint length_pack; + const int reverse = keyinfo->seg->flag & HA_REVERSE_SORT; + DBUG_ENTER("_mi_prefix_search"); + + t_buff[0]=0; /* Avoid bugs */ + end= page+mi_getint(page); + nod_flag=mi_test_if_nod(page); + page+=2+nod_flag; + *ret_pos=page; + kseg=key; + + get_key_pack_length(kseg_len,length_pack,kseg); + key_len_skip=length_pack+kseg_len; + key_len_left=(int) key_len- (int) key_len_skip; + /* If key_len is 0, then lenght_pack is 1, then key_len_left is -1. */ + cmplen=(key_len_left>=0) ? kseg_len : key_len-length_pack; + DBUG_PRINT("info",("key: '%.*s'",kseg_len,kseg)); + + /* + Keys are compressed the following way: + + If the max length of first key segment <= 127 bytes the prefix is + 1 byte else it's 2 byte + + (prefix) length The high bit is set if this is a prefix for the prev key. + [suffix length] Packed length of suffix if the previous was a prefix. + (suffix) data Key data bytes (past the common prefix or whole segment). + [next-key-seg] Next key segments (([packed length], data), ...) + pointer Reference to the data file (last_keyseg->length). + */ + + matched=0; /* how many char's from prefix were alredy matched */ + len=0; /* length of previous key unpacked */ + + while (page < end) + { + uint packed= *page & 128; + + vseg=page; + if (keyinfo->seg->length >= 127) + { + suffix_len=mi_uint2korr(vseg) & 32767; + vseg+=2; + } + else + suffix_len= *vseg++ & 127; + + if (packed) + { + if (suffix_len == 0) + { + /* == 0x80 or 0x8000, same key, prefix length == old key length. */ + prefix_len=len; + } + else + { + /* > 0x80 or 0x8000, this is prefix lgt, packed suffix lgt follows. */ + prefix_len=suffix_len; + get_key_length(suffix_len,vseg); + } + } + else + { + /* Not packed. No prefix used from last key. */ + prefix_len=0; + } + + len=prefix_len+suffix_len; + seg_len_pack=get_pack_length(len); + t_buff=tt_buff+3-seg_len_pack; + store_key_length(t_buff,len); + + if (prefix_len > saved_prefix_len) + memcpy(t_buff+seg_len_pack+saved_prefix_len,saved_vseg, + prefix_len-saved_prefix_len); + saved_vseg=vseg; + saved_prefix_len=prefix_len; + + DBUG_PRINT("loop",("page: '%.*s%.*s'",prefix_len,t_buff+seg_len_pack, + suffix_len,vseg)); + { + uchar *from=vseg+suffix_len; + HA_KEYSEG *keyseg; + uint l; + + for (keyseg=keyinfo->seg+1 ; keyseg->type ; keyseg++ ) + { + + if (keyseg->flag & HA_NULL_PART) + { + if (!(*from++)) + continue; + } + if (keyseg->flag & (HA_VAR_LENGTH_PART | HA_BLOB_PART | HA_SPACE_PACK)) + { + get_key_length(l,from); + } + else + l=keyseg->length; + + from+=l; + } + from+=keyseg->length; + page=from+nod_flag; + length= (uint) (from - vseg); + } + + if (page > end) + { + mi_print_error(info->s, HA_ERR_CRASHED); + my_errno=HA_ERR_CRASHED; + DBUG_PRINT("error", + ("Found wrong key: length: %u page: %p end: %p", + length, page, end)); + DBUG_RETURN(MI_FOUND_WRONG_KEY); + } + + if (matched >= prefix_len) + { + /* We have to compare. But we can still skip part of the key */ + uint left; + uchar *k=kseg+prefix_len; + + /* + If prefix_len > cmplen then we are in the end-space comparison + phase. Do not try to access the key any more ==> left= 0. + */ + left= ((len <= cmplen) ? suffix_len : + ((prefix_len < cmplen) ? cmplen - prefix_len : 0)); + + matched=prefix_len+left; + + if (sort_order) + { + for (my_flag=0;left;left--) + if ((my_flag= (int) sort_order[*vseg++] - (int) sort_order[*k++])) + break; + } + else + { + for (my_flag=0;left;left--) + if ((my_flag= (int) *vseg++ - (int) *k++)) + break; + } + + if (my_flag==0) /* match */ + { + /* + ** len cmplen seg_left_len more_segs + ** < matched=len; continue search + ** > = prefix ? found : (matched=len; continue search) + ** > < - ok, found + ** = < - ok, found + ** = = - ok, found + ** = = + next seg + */ + if (len < cmplen) + { + if ((keyinfo->seg->type != HA_KEYTYPE_TEXT && + keyinfo->seg->type != HA_KEYTYPE_VARTEXT1 && + keyinfo->seg->type != HA_KEYTYPE_VARTEXT2)) + my_flag= -1; + else + { + /* We have to compare k and vseg as if they were space extended */ + uchar *k_end= k+ (cmplen - len); + for ( ; k < k_end && *k == ' '; k++) ; + if (k == k_end) + goto cmp_rest; /* should never happen */ + my_flag= (uchar)' ' - *k; + } + } + else if (len > cmplen) + { + uchar *vseg_end; + if ((nextflag & SEARCH_PREFIX) && key_len_left == 0) + goto fix_flag; + + /* We have to compare k and vseg as if they were space extended */ + for (vseg_end= vseg + (len-cmplen) ; + vseg < vseg_end && *vseg == (uchar) ' '; + vseg++, matched++) ; + DBUG_ASSERT(vseg < vseg_end); + + my_flag= *vseg - (uchar)' '; + } + else + { + cmp_rest: + if (key_len_left>0) + { + uint not_used[2]; + if ((flag = ha_key_cmp(keyinfo->seg+1,vseg, + k, key_len_left, nextflag, not_used)) >= 0) + break; + } + else + { + /* + at this line flag==-1 if the following lines were already + visited and 0 otherwise, i.e. flag <=0 here always !!! + */ + fix_flag: + DBUG_ASSERT(flag <= 0); + if (nextflag & (SEARCH_NO_FIND | SEARCH_LAST)) + flag=(nextflag & (SEARCH_BIGGER | SEARCH_LAST)) ? -1 : 1; + if (flag>=0) + break; + } + } + } + if ((reverse ? -my_flag : my_flag) > 0) /* mismatch */ + break; + matched-=left; + } + /* else (matched < prefix_len) ---> do nothing. */ + + memcpy(buff,t_buff,saved_length=seg_len_pack+prefix_len); + saved_to=buff+saved_length; + saved_from=saved_vseg; + saved_length=length; + *ret_pos=page; + } + if (my_flag) + flag= reverse ? -my_flag : my_flag; + if (flag == 0) + { + memcpy(buff,t_buff,saved_length=seg_len_pack+prefix_len); + saved_to=buff+saved_length; + saved_from=saved_vseg; + saved_length=length; + } + if (saved_length) + memcpy(saved_to,saved_from,saved_length); + + *last_key= page == end; + + DBUG_PRINT("exit",("flag: %d ret_pos: %p", flag, *ret_pos)); + DBUG_RETURN(flag); +} /* _mi_prefix_search */ + + + /* Get pos to a key_block */ + +my_off_t _mi_kpos(uint nod_flag, uchar *after_key) +{ + after_key-=nod_flag; + switch (nod_flag) { +#if SIZEOF_OFF_T > 4 + case 7: + return mi_uint7korr(after_key)*MI_MIN_KEY_BLOCK_LENGTH; + case 6: + return mi_uint6korr(after_key)*MI_MIN_KEY_BLOCK_LENGTH; + case 5: + return mi_uint5korr(after_key)*MI_MIN_KEY_BLOCK_LENGTH; +#else + case 7: + after_key++; + case 6: + after_key++; + case 5: + after_key++; +#endif + case 4: + return ((my_off_t) mi_uint4korr(after_key))*MI_MIN_KEY_BLOCK_LENGTH; + case 3: + return ((my_off_t) mi_uint3korr(after_key))*MI_MIN_KEY_BLOCK_LENGTH; + case 2: + return (my_off_t) (mi_uint2korr(after_key)*MI_MIN_KEY_BLOCK_LENGTH); + case 1: + return (uint) (*after_key)*MI_MIN_KEY_BLOCK_LENGTH; + case 0: /* At leaf page */ + default: /* Impossible */ + return(HA_OFFSET_ERROR); + } +} /* _kpos */ + + + /* Save pos to a key_block */ + +void _mi_kpointer(register MI_INFO *info, register uchar *buff, my_off_t pos) +{ + pos/=MI_MIN_KEY_BLOCK_LENGTH; + switch (info->s->base.key_reflength) { +#if SIZEOF_OFF_T > 4 + case 7: mi_int7store(buff,pos); break; + case 6: mi_int6store(buff,pos); break; + case 5: mi_int5store(buff,pos); break; +#else + case 7: *buff++=0; + /* fall through */ + case 6: *buff++=0; + /* fall through */ + case 5: *buff++=0; + /* fall through */ +#endif + case 4: mi_int4store(buff,pos); break; + case 3: mi_int3store(buff,pos); break; + case 2: mi_int2store(buff,(uint) pos); break; + case 1: buff[0]= (uchar) pos; break; + default: abort(); /* impossible */ + } +} /* _mi_kpointer */ + + + /* Calc pos to a data-record from a key */ + + +my_off_t _mi_dpos(MI_INFO *info, uint nod_flag, uchar *after_key) +{ + my_off_t pos; + after_key-=(nod_flag + info->s->rec_reflength); + switch (info->s->rec_reflength) { +#if SIZEOF_OFF_T > 4 + case 8: pos= (my_off_t) mi_uint8korr(after_key); break; + case 7: pos= (my_off_t) mi_uint7korr(after_key); break; + case 6: pos= (my_off_t) mi_uint6korr(after_key); break; + case 5: pos= (my_off_t) mi_uint5korr(after_key); break; +#else + case 8: pos= (my_off_t) mi_uint4korr(after_key+4); break; + case 7: pos= (my_off_t) mi_uint4korr(after_key+3); break; + case 6: pos= (my_off_t) mi_uint4korr(after_key+2); break; + case 5: pos= (my_off_t) mi_uint4korr(after_key+1); break; +#endif + case 4: pos= (my_off_t) mi_uint4korr(after_key); break; + case 3: pos= (my_off_t) mi_uint3korr(after_key); break; + case 2: pos= (my_off_t) mi_uint2korr(after_key); break; + default: + pos=0L; /* Shut compiler up */ + } + return (info->s->options & + (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ? pos : + pos*info->s->base.pack_reclength; +} + + +/* Calc position from a record pointer ( in delete link chain ) */ + +my_off_t _mi_rec_pos(MYISAM_SHARE *s, uchar *ptr) +{ + my_off_t pos; + switch (s->rec_reflength) { +#if SIZEOF_OFF_T > 4 + case 8: + pos= (my_off_t) mi_uint8korr(ptr); + if (pos == HA_OFFSET_ERROR) + return HA_OFFSET_ERROR; /* end of list */ + break; + case 7: + pos= (my_off_t) mi_uint7korr(ptr); + if (pos == (((my_off_t) 1) << 56) -1) + return HA_OFFSET_ERROR; /* end of list */ + break; + case 6: + pos= (my_off_t) mi_uint6korr(ptr); + if (pos == (((my_off_t) 1) << 48) -1) + return HA_OFFSET_ERROR; /* end of list */ + break; + case 5: + pos= (my_off_t) mi_uint5korr(ptr); + if (pos == (((my_off_t) 1) << 40) -1) + return HA_OFFSET_ERROR; /* end of list */ + break; +#else + case 8: + case 7: + case 6: + case 5: + ptr+= (s->rec_reflength-4); + /* fall through */ +#endif + case 4: + pos= (my_off_t) mi_uint4korr(ptr); + if (pos == (my_off_t) (uint32) ~0L) + return HA_OFFSET_ERROR; + break; + case 3: + pos= (my_off_t) mi_uint3korr(ptr); + if (pos == (my_off_t) (1 << 24) -1) + return HA_OFFSET_ERROR; + break; + case 2: + pos= (my_off_t) mi_uint2korr(ptr); + if (pos == (my_off_t) (1 << 16) -1) + return HA_OFFSET_ERROR; + break; + default: abort(); /* Impossible */ + } + return ((s->options & + (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ? pos : + pos*s->base.pack_reclength); +} + + + /* save position to record */ + +void _mi_dpointer(MI_INFO *info, uchar *buff, my_off_t pos) +{ + if (!(info->s->options & + (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) && + pos != HA_OFFSET_ERROR) + pos/=info->s->base.pack_reclength; + + switch (info->s->rec_reflength) { +#if SIZEOF_OFF_T > 4 + case 8: mi_int8store(buff,pos); break; + case 7: mi_int7store(buff,pos); break; + case 6: mi_int6store(buff,pos); break; + case 5: mi_int5store(buff,pos); break; +#else + case 8: *buff++=0; + /* fall through */ + case 7: *buff++=0; + /* fall through */ + case 6: *buff++=0; + /* fall through */ + case 5: *buff++=0; + /* fall through */ +#endif + case 4: mi_int4store(buff,pos); break; + case 3: mi_int3store(buff,pos); break; + case 2: mi_int2store(buff,(uint) pos); break; + default: abort(); /* Impossible */ + } +} /* _mi_dpointer */ + + + /* Get key from key-block */ + /* page points at previous key; its advanced to point at next key */ + /* key should contain previous key */ + /* Returns length of found key + pointers */ + /* nod_flag is a flag if we are on nod */ + + /* same as _mi_get_key but used with fixed length keys */ + +uint _mi_get_static_key(register MI_KEYDEF *keyinfo, uint nod_flag, + register uchar **page, register uchar *key) +{ + memcpy((uchar*) key,(uchar*) *page, + (size_t) (keyinfo->keylength+nod_flag)); + *page+=keyinfo->keylength+nod_flag; + return(keyinfo->keylength); +} /* _mi_get_static_key */ + + +/* + get key witch is packed against previous key or key with a NULL column. + + SYNOPSIS + _mi_get_pack_key() + keyinfo key definition information. + nod_flag If nod: Length of node pointer, else zero. + page_pos RETURN position in key page behind this key. + key IN/OUT in: prev key, out: unpacked key. + + RETURN + key_length + length of data pointer +*/ + +uint _mi_get_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag, + register uchar **page_pos, register uchar *key) +{ + reg1 HA_KEYSEG *keyseg; + uchar *start_key,*page=*page_pos; + uint length; + + start_key=key; + for (keyseg=keyinfo->seg ; keyseg->type ;keyseg++) + { + if (keyseg->flag & HA_PACK_KEY) + { + /* key with length, packed to previous key */ + uchar *start=key; + uint packed= *page & 128,tot_length,rest_length; + if (keyseg->length >= 127) + { + length=mi_uint2korr(page) & 32767; + page+=2; + } + else + length= *page++ & 127; + + if (packed) + { + if (length > (uint) keyseg->length) + { + mi_print_error(keyinfo->share, HA_ERR_CRASHED); + my_errno=HA_ERR_CRASHED; + return 0; /* Error */ + } + if (length == 0) /* Same key */ + { + if (keyseg->flag & HA_NULL_PART) + *key++=1; /* Can't be NULL */ + get_key_length(length,key); + key+= length; /* Same diff_key as prev */ + if (length > keyseg->length) + { + DBUG_PRINT("error", + ("Found too long null packed key: %u of %u at %p", + length, keyseg->length, *page_pos)); + DBUG_DUMP("key", *page_pos, 16); + mi_print_error(keyinfo->share, HA_ERR_CRASHED); + my_errno=HA_ERR_CRASHED; + return 0; + } + continue; + } + if (keyseg->flag & HA_NULL_PART) + { + key++; /* Skip null marker*/ + start++; + } + + get_key_length(rest_length,page); + tot_length=rest_length+length; + + /* If the stored length has changed, we must move the key */ + if (tot_length >= 255 && *start != 255) + { + /* length prefix changed from a length of one to a length of 3 */ + bmove_upp(key+length+3, key+length+1, length); + *key=255; + mi_int2store(key+1,tot_length); + key+=3+length; + } + else if (tot_length < 255 && *start == 255) + { + bmove(key+1,key+3,length); + *key=tot_length; + key+=1+length; + } + else + { + store_key_length_inc(key,tot_length); + key+=length; + } + memcpy(key,page,rest_length); + page+=rest_length; + key+=rest_length; + continue; + } + else + { + if (keyseg->flag & HA_NULL_PART) + { + if (!length--) /* Null part */ + { + *key++=0; + continue; + } + *key++=1; /* Not null */ + } + } + if (length > (uint) keyseg->length) + { + DBUG_PRINT("error",("Found too long packed key: %u of %u at %p", + length, keyseg->length, *page_pos)); + DBUG_DUMP("key", *page_pos, 16); + mi_print_error(keyinfo->share, HA_ERR_CRASHED); + my_errno=HA_ERR_CRASHED; + return 0; /* Error */ + } + store_key_length_inc(key,length); + } + else + { + if (keyseg->flag & HA_NULL_PART) + { + if (!(*key++ = *page++)) + continue; + } + if (keyseg->flag & + (HA_VAR_LENGTH_PART | HA_BLOB_PART | HA_SPACE_PACK)) + { + uchar *tmp=page; + get_key_length(length,tmp); + length+=(uint) (tmp-page); + } + else + length=keyseg->length; + } + memcpy((uchar*) key,(uchar*) page,(size_t) length); + key+=length; + page+=length; + } + length=keyseg->length+nod_flag; + bmove((uchar*) key,(uchar*) page,length); + *page_pos= page+length; + return ((uint) (key-start_key)+keyseg->length); +} /* _mi_get_pack_key */ + + + +/* key that is packed relatively to previous */ + +uint _mi_get_binary_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag, + register uchar **page_pos, register uchar *key) +{ + reg1 HA_KEYSEG *keyseg; + uchar *start_key,*page,*page_end,*from,*from_end; + uint length,tmp; + DBUG_ENTER("_mi_get_binary_pack_key"); + + page= *page_pos; + page_end=page+HA_MAX_KEY_BUFF+1; + start_key=key; + + /* + Keys are compressed the following way: + + prefix length Packed length of prefix common with prev key (1 or 3 bytes) + for each key segment: + [is null] Null indicator if can be null (1 byte, zero means null) + [length] Packed length if varlength (1 or 3 bytes) + key segment 'length' bytes of key segment value + pointer Reference to the data file (last_keyseg->length). + + get_key_length() is a macro. It gets the prefix length from 'page' + and puts it into 'length'. It increments 'page' by 1 or 3, depending + on the packed length of the prefix length. + */ + get_key_length(length,page); + if (length) + { + if (length > keyinfo->maxlength) + { + DBUG_PRINT("error", + ("Found too long binary packed key: %u of %u at %p", + length, keyinfo->maxlength, *page_pos)); + DBUG_DUMP("key", *page_pos, 16); + goto crashed; /* Wrong key */ + } + /* Key is packed against prev key, take prefix from prev key. */ + from= key; + from_end= key + length; + } + else + { + /* Key is not packed against prev key, take all from page buffer. */ + from= page; + from_end= page_end; + } + + /* + The trouble is that key can be split in two parts: + The first part (prefix) is in from .. from_end - 1. + The second part starts at page. + The split can be at every byte position. So we need to check for + the end of the first part before using every byte. + */ + for (keyseg=keyinfo->seg ; keyseg->type ;keyseg++) + { + if (keyseg->flag & HA_NULL_PART) + { + /* If prefix is used up, switch to rest. */ + if (from == from_end) { from=page; from_end=page_end; } + if (!(*key++ = *from++)) + continue; /* Null part */ + } + if (keyseg->flag & (HA_VAR_LENGTH_PART | HA_BLOB_PART | HA_SPACE_PACK)) + { + /* If prefix is used up, switch to rest. */ + if (from == from_end) { from=page; from_end=page_end; } + /* Get length of dynamic length key part */ + if ((length= (*key++ = *from++)) == 255) + { + /* If prefix is used up, switch to rest. */ + if (from == from_end) { from=page; from_end=page_end; } + length= (uint) ((*key++ = *from++)) << 8; + /* If prefix is used up, switch to rest. */ + if (from == from_end) { from=page; from_end=page_end; } + length+= (uint) ((*key++ = *from++)); + } + if (length > keyseg->length) + goto crashed; + } + else + length=keyseg->length; + + if ((tmp=(uint) (from_end-from)) <= length) + { + key+=tmp; /* Use old key */ + length-=tmp; + from=page; from_end=page_end; + } + DBUG_PRINT("info",("key: %p from: %p length: %u", + key, from, length)); + memmove((uchar*) key, (uchar*) from, (size_t) length); + key+=length; + from+=length; + } + /* + Last segment (type == 0) contains length of data pointer. + If we have mixed key blocks with data pointer and key block pointer, + we have to copy both. + */ + length=keyseg->length+nod_flag; + if ((tmp=(uint) (from_end-from)) <= length) + { + /* Remaining length is less or equal max possible length. */ + memcpy(key+tmp,page,length-tmp); /* Get last part of key */ + *page_pos= page+length-tmp; + } + else + { + /* + Remaining length is greater than max possible length. + This can happen only if we switched to the new key bytes already. + 'page_end' is calculated with MI_MAX_KEY_BUFF. So it can be far + behind the real end of the key. + */ + if (from_end != page_end) + { + DBUG_PRINT("error",("Error when unpacking key")); + goto crashed; /* Error */ + } + /* Copy data pointer and, if appropriate, key block pointer. */ + memcpy((uchar*) key,(uchar*) from,(size_t) length); + *page_pos= from+length; + } + DBUG_RETURN((uint) (key-start_key)+keyseg->length); + + crashed: + mi_print_error(keyinfo->share, HA_ERR_CRASHED); + my_errno= HA_ERR_CRASHED; + DBUG_RETURN(0); +} + + + /* Get key at position without knowledge of previous key */ + /* Returns pointer to next key */ + +uchar *_mi_get_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, + uchar *key, uchar *keypos, uint *return_key_length) +{ + uint nod_flag; + DBUG_ENTER("_mi_get_key"); + + nod_flag=mi_test_if_nod(page); + if (! (keyinfo->flag & (HA_VAR_LENGTH_KEY | HA_BINARY_PACK_KEY))) + { + bmove((uchar*) key,(uchar*) keypos,keyinfo->keylength+nod_flag); + DBUG_RETURN(keypos+keyinfo->keylength+nod_flag); + } + else + { + page+=2+nod_flag; + key[0]=0; /* safety */ + while (page <= keypos) + { + *return_key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&page,key); + if (*return_key_length == 0) + { + mi_print_error(info->s, HA_ERR_CRASHED); + my_errno=HA_ERR_CRASHED; + DBUG_RETURN(0); + } + } + } + DBUG_PRINT("exit",("page: %p length: %u", page, + *return_key_length)); + DBUG_RETURN(page); +} /* _mi_get_key */ + + + /* Get key at position without knowledge of previous key */ + /* Returns 0 if ok */ + +static my_bool _mi_get_prev_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, + uchar *key, uchar *keypos, + uint *return_key_length) +{ + uint nod_flag; + DBUG_ENTER("_mi_get_prev_key"); + + nod_flag=mi_test_if_nod(page); + if (! (keyinfo->flag & (HA_VAR_LENGTH_KEY | HA_BINARY_PACK_KEY))) + { + *return_key_length=keyinfo->keylength; + bmove((uchar*) key,(uchar*) keypos- *return_key_length-nod_flag, + *return_key_length); + DBUG_RETURN(0); + } + else + { + page+=2+nod_flag; + key[0]=0; /* safety */ + while (page < keypos) + { + *return_key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&page,key); + if (*return_key_length == 0) + { + mi_print_error(info->s, HA_ERR_CRASHED); + my_errno=HA_ERR_CRASHED; + DBUG_RETURN(1); + } + } + } + DBUG_RETURN(0); +} /* _mi_get_key */ + + + + /* Get last key from key-page */ + /* Return pointer to where key starts */ + +uchar *_mi_get_last_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, + uchar *lastkey, uchar *endpos, uint *return_key_length) +{ + uint nod_flag; + uchar *lastpos; + DBUG_ENTER("_mi_get_last_key"); + DBUG_PRINT("enter",("page:%p endpos: %p", page, + endpos)); + + nod_flag=mi_test_if_nod(page); + if (! (keyinfo->flag & (HA_VAR_LENGTH_KEY | HA_BINARY_PACK_KEY))) + { + lastpos=endpos-keyinfo->keylength-nod_flag; + *return_key_length=keyinfo->keylength; + if (lastpos > page) + bmove((uchar*) lastkey,(uchar*) lastpos,keyinfo->keylength+nod_flag); + } + else + { + lastpos=(page+=2+nod_flag); + lastkey[0]=0; + while (page < endpos) + { + lastpos=page; + *return_key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&page,lastkey); + if (*return_key_length == 0) + { + DBUG_PRINT("error",("Couldn't find last key: page: %p", + page)); + mi_print_error(info->s, HA_ERR_CRASHED); + my_errno=HA_ERR_CRASHED; + DBUG_RETURN(0); + } + } + } + DBUG_PRINT("exit",("lastpos: %p length: %u", lastpos, + *return_key_length)); + DBUG_RETURN(lastpos); +} /* _mi_get_last_key */ + + + /* Calculate length of key */ + +uint _mi_keylength(MI_KEYDEF *keyinfo, register uchar *key) +{ + reg1 HA_KEYSEG *keyseg; + uchar *start; + + if (! (keyinfo->flag & (HA_VAR_LENGTH_KEY | HA_BINARY_PACK_KEY))) + return (keyinfo->keylength); + + start=key; + for (keyseg=keyinfo->seg ; keyseg->type ; keyseg++) + { + if (keyseg->flag & HA_NULL_PART) + if (!*key++) + continue; + if (keyseg->flag & (HA_SPACE_PACK | HA_BLOB_PART | HA_VAR_LENGTH_PART)) + { + uint length; + get_key_length(length,key); + key+=length; + } + else + key+= keyseg->length; + } + return((uint) (key-start)+keyseg->length); +} /* _mi_keylength */ + + +/* + Calculate length of part key. + + Used in mi_rkey() to find the key found for the key-part that was used. + This is needed in case of multi-byte character sets where we may search + after '0xDF' but find 'ss' +*/ + +uint _mi_keylength_part(MI_KEYDEF *keyinfo, register uchar *key, + HA_KEYSEG *end) +{ + reg1 HA_KEYSEG *keyseg; + uchar *start= key; + + for (keyseg=keyinfo->seg ; keyseg != end ; keyseg++) + { + if (keyseg->flag & HA_NULL_PART) + if (!*key++) + continue; + if (keyseg->flag & (HA_SPACE_PACK | HA_BLOB_PART | HA_VAR_LENGTH_PART)) + { + uint length; + get_key_length(length,key); + key+=length; + } + else + key+= keyseg->length; + } + return (uint) (key-start); +} + + /* Move a key */ + +uchar *_mi_move_key(MI_KEYDEF *keyinfo, uchar *to, uchar *from) +{ + reg1 uint length; + memcpy((uchar*) to, (uchar*) from, + (size_t) (length=_mi_keylength(keyinfo,from))); + return to+length; +} + + /* Find next/previous record with same key */ + /* This can't be used when database is touched after last read */ + +int _mi_search_next(register MI_INFO *info, register MI_KEYDEF *keyinfo, + uchar *key, uint key_length, uint nextflag, my_off_t pos) +{ + int error; + uint nod_flag; + uchar lastkey[HA_MAX_KEY_BUFF]; + DBUG_ENTER("_mi_search_next"); + DBUG_PRINT("enter",("nextflag: %u lastpos: %llu int_keypos: %p", + nextflag, info->lastpos, + info->int_keypos)); + DBUG_EXECUTE("key",_mi_print_key(DBUG_FILE,keyinfo->seg,key,key_length);); + + /* Force full read if we are at last key or if we are not on a leaf + and the key tree has changed since we used it last time + Note that even if the key tree has changed since last read, we can use + the last read data from the leaf if we haven't used the buffer for + something else. + */ + + if (((nextflag & SEARCH_BIGGER) && info->int_keypos >= info->int_maxpos) || + info->page_changed || + (info->int_keytree_version != keyinfo->version && + (info->int_nod_flag || info->buff_used))) + DBUG_RETURN(_mi_search(info,keyinfo,key, USE_WHOLE_KEY, + nextflag | SEARCH_SAVE_BUFF, pos)); + + if (info->buff_used) + { + if (!_mi_fetch_keypage(info,keyinfo,info->last_search_keypage, + DFLT_INIT_HITS,info->buff,0)) + DBUG_RETURN(-1); + info->buff_used=0; + } + + /* Last used buffer is in info->buff */ + nod_flag=mi_test_if_nod(info->buff); + + if (nextflag & SEARCH_BIGGER) /* Next key */ + { + my_off_t tmp_pos=_mi_kpos(nod_flag,info->int_keypos); + if (tmp_pos != HA_OFFSET_ERROR) + { + if ((error=_mi_search(info,keyinfo,key, USE_WHOLE_KEY, + nextflag | SEARCH_SAVE_BUFF, tmp_pos)) <=0) + DBUG_RETURN(error); + } + memcpy(lastkey,key,key_length); + if (!(info->lastkey_length=(*keyinfo->get_key)(keyinfo,nod_flag, + &info->int_keypos,lastkey))) + DBUG_RETURN(-1); + } + else /* Previous key */ + { + uint length; + /* Find start of previous key */ + info->int_keypos=_mi_get_last_key(info,keyinfo,info->buff,lastkey, + info->int_keypos, &length); + if (!info->int_keypos) + DBUG_RETURN(-1); + if (info->int_keypos == info->buff+2) + DBUG_RETURN(_mi_search(info,keyinfo,key, USE_WHOLE_KEY, + nextflag | SEARCH_SAVE_BUFF, pos)); + if ((error=_mi_search(info,keyinfo,key, USE_WHOLE_KEY, + nextflag | SEARCH_SAVE_BUFF, + _mi_kpos(nod_flag,info->int_keypos))) <= 0) + DBUG_RETURN(error); + + /* QQ: We should be able to optimize away the following call */ + if (! _mi_get_last_key(info,keyinfo,info->buff,lastkey, + info->int_keypos,&info->lastkey_length)) + DBUG_RETURN(-1); + } + memcpy(info->lastkey,lastkey,info->lastkey_length); + info->lastpos=_mi_dpos(info,0,info->lastkey+info->lastkey_length); + DBUG_PRINT("exit",("found key at %lu",(ulong) info->lastpos)); + DBUG_RETURN(0); +} /* _mi_search_next */ + + + /* Search after position for the first row in an index */ + /* This is stored in info->lastpos */ + +int _mi_search_first(register MI_INFO *info, register MI_KEYDEF *keyinfo, + register my_off_t pos) +{ + uint nod_flag; + uchar *page; + DBUG_ENTER("_mi_search_first"); + + if (pos == HA_OFFSET_ERROR) + { + my_errno=HA_ERR_KEY_NOT_FOUND; + info->lastpos= HA_OFFSET_ERROR; + DBUG_RETURN(-1); + } + + do + { + if (!_mi_fetch_keypage(info,keyinfo,pos,DFLT_INIT_HITS,info->buff,0)) + { + info->lastpos= HA_OFFSET_ERROR; + DBUG_RETURN(-1); + } + nod_flag=mi_test_if_nod(info->buff); + page=info->buff+2+nod_flag; + } while ((pos=_mi_kpos(nod_flag,page)) != HA_OFFSET_ERROR); + + if (!(info->lastkey_length=(*keyinfo->get_key)(keyinfo,nod_flag,&page, + info->lastkey))) + DBUG_RETURN(-1); /* Crashed */ + + info->int_keypos=page; info->int_maxpos=info->buff+mi_getint(info->buff)-1; + info->int_nod_flag=nod_flag; + info->int_keytree_version=keyinfo->version; + info->last_search_keypage=info->last_keypage; + info->page_changed=info->buff_used=0; + info->lastpos=_mi_dpos(info,0,info->lastkey+info->lastkey_length); + + DBUG_PRINT("exit",("found key at %lu", (ulong) info->lastpos)); + DBUG_RETURN(0); +} /* _mi_search_first */ + + + /* Search after position for the last row in an index */ + /* This is stored in info->lastpos */ + +int _mi_search_last(register MI_INFO *info, register MI_KEYDEF *keyinfo, + register my_off_t pos) +{ + uint nod_flag; + uchar *buff,*page; + DBUG_ENTER("_mi_search_last"); + + if (pos == HA_OFFSET_ERROR) + { + my_errno=HA_ERR_KEY_NOT_FOUND; /* Didn't find key */ + info->lastpos= HA_OFFSET_ERROR; + DBUG_RETURN(-1); + } + + buff=info->buff; + do + { + if (!_mi_fetch_keypage(info,keyinfo,pos,DFLT_INIT_HITS,buff,0)) + { + info->lastpos= HA_OFFSET_ERROR; + DBUG_RETURN(-1); + } + page= buff+mi_getint(buff); + nod_flag=mi_test_if_nod(buff); + } while ((pos=_mi_kpos(nod_flag,page)) != HA_OFFSET_ERROR); + + if (!_mi_get_last_key(info,keyinfo,buff,info->lastkey,page, + &info->lastkey_length)) + DBUG_RETURN(-1); + info->lastpos=_mi_dpos(info,0,info->lastkey+info->lastkey_length); + info->int_keypos=info->int_maxpos=page; + info->int_nod_flag=nod_flag; + info->int_keytree_version=keyinfo->version; + info->last_search_keypage=info->last_keypage; + info->page_changed=info->buff_used=0; + + DBUG_PRINT("exit",("found key at %lu",(ulong) info->lastpos)); + DBUG_RETURN(0); +} /* _mi_search_last */ + + + +/**************************************************************************** +** +** Functions to store and pack a key in a page +** +** mi_calc_xx_key_length takes the following arguments: +** nod_flag If nod: Length of nod-pointer +** next_key Position to pos after the new key in buffer +** org_key Key that was before the next key in buffer +** prev_key Last key before current key +** key Key that will be stored +** s_temp Information how next key will be packed +****************************************************************************/ + +/* Static length key */ + +int +_mi_calc_static_key_length(MI_KEYDEF *keyinfo,uint nod_flag, + uchar *next_pos __attribute__((unused)), + uchar *org_key __attribute__((unused)), + uchar *prev_key __attribute__((unused)), + uchar *key, MI_KEY_PARAM *s_temp) +{ + s_temp->key=key; + return (int) (s_temp->totlength=keyinfo->keylength+nod_flag); +} + +/* Variable length key */ + +int +_mi_calc_var_key_length(MI_KEYDEF *keyinfo,uint nod_flag, + uchar *next_pos __attribute__((unused)), + uchar *org_key __attribute__((unused)), + uchar *prev_key __attribute__((unused)), + uchar *key, MI_KEY_PARAM *s_temp) +{ + s_temp->key=key; + return (int) (s_temp->totlength=_mi_keylength(keyinfo,key)+nod_flag); +} + +/* + length of key with a variable length first segment which is prefix + compressed (myisamchk reports 'packed + stripped') + + Keys are compressed the following way: + + If the max length of first key segment <= 127 bytes the prefix is + 1 byte else it's 2 byte + + prefix byte(s) The high bit is set if this is a prefix for the prev key + length Packed length if the previous was a prefix byte + [length] data bytes ('length' bytes) + next-key-seg Next key segments + + If the first segment can have NULL: + The length is 0 for NULLS and 1+length for not null columns. + +*/ + +int +_mi_calc_var_pack_key_length(MI_KEYDEF *keyinfo,uint nod_flag,uchar *next_key, + uchar *org_key, uchar *prev_key, uchar *key, + MI_KEY_PARAM *s_temp) +{ + reg1 HA_KEYSEG *keyseg; + int length; + uint key_length,ref_length,org_key_length=0, + length_pack,new_key_length,diff_flag,pack_marker; + uchar *start,*end,*key_end; + const uchar *sort_order; + my_bool same_length; + + length_pack=s_temp->ref_length=s_temp->n_ref_length=s_temp->n_length=0; + same_length=0; keyseg=keyinfo->seg; + key_length=_mi_keylength(keyinfo,key)+nod_flag; + + sort_order=0; + if ((keyinfo->flag & HA_FULLTEXT) && + ((keyseg->type == HA_KEYTYPE_TEXT) || + (keyseg->type == HA_KEYTYPE_VARTEXT1) || + (keyseg->type == HA_KEYTYPE_VARTEXT2)) && + !use_strnxfrm(keyseg->charset)) + sort_order=keyseg->charset->sort_order; + + /* diff flag contains how many bytes is needed to pack key */ + if (keyseg->length >= 127) + { + diff_flag=2; + pack_marker=32768; + } + else + { + diff_flag= 1; + pack_marker=128; + } + s_temp->pack_marker=pack_marker; + + /* Handle the case that the first part have NULL values */ + if (keyseg->flag & HA_NULL_PART) + { + if (!*key++) + { + s_temp->key=key; + s_temp->key_length= 0; + s_temp->totlength=key_length-1+diff_flag; + s_temp->next_key_pos=0; /* No next key */ + return (s_temp->totlength); + } + s_temp->store_not_null=1; + key_length--; /* We don't store NULL */ + if (prev_key && !*prev_key++) + org_key=prev_key=0; /* Can't pack against prev */ + else if (org_key) + org_key++; /* Skip NULL */ + } + else + s_temp->store_not_null=0; + s_temp->prev_key=org_key; + + /* The key part will start with a packed length */ + + get_key_pack_length(new_key_length,length_pack,key); + end=key_end= key+ new_key_length; + start=key; + + /* Calc how many characters are identical between this and the prev. key */ + if (prev_key) + { + get_key_length(org_key_length,prev_key); + s_temp->prev_key=prev_key; /* Pointer at data */ + /* Don't use key-pack if length == 0 */ + if (new_key_length && new_key_length == org_key_length) + same_length=1; + else if (new_key_length > org_key_length) + end=key + org_key_length; + + if (sort_order) /* SerG */ + { + while (key < end && sort_order[*key] == sort_order[*prev_key]) + { + key++; prev_key++; + } + } + else + { + while (key < end && *key == *prev_key) + { + key++; prev_key++; + } + } + } + + s_temp->key=key; + s_temp->key_length= (uint) (key_end-key); + + if (same_length && key == key_end) + { + /* identical variable length key */ + s_temp->ref_length= pack_marker; + length=(int) key_length-(int) (key_end-start)-length_pack; + length+= diff_flag; + if (next_key) + { /* Can't combine with next */ + s_temp->n_length= *next_key; /* Needed by _mi_store_key */ + next_key=0; + } + } + else + { + if (start != key) + { /* Starts as prev key */ + ref_length= (uint) (key-start); + s_temp->ref_length= ref_length + pack_marker; + length= (int) (key_length - ref_length); + + length-= length_pack; + length+= diff_flag; + length+= ((new_key_length-ref_length) >= 255) ? 3 : 1;/* Rest_of_key */ + } + else + { + s_temp->key_length+=s_temp->store_not_null; /* If null */ + length= key_length - length_pack+ diff_flag; + } + } + s_temp->totlength=(uint) length; + s_temp->prev_length=0; + DBUG_PRINT("test",("tot_length: %u length: %d uniq_key_length: %u", + key_length, length, s_temp->key_length)); + + /* If something after that hasn't length=0, test if we can combine */ + if ((s_temp->next_key_pos=next_key)) + { + uint packed,n_length; + + packed = *next_key & 128; + if (diff_flag == 2) + { + n_length= mi_uint2korr(next_key) & 32767; /* Length of next key */ + next_key+=2; + } + else + n_length= *next_key++ & 127; + if (!packed && n_length) + n_length-= s_temp->store_not_null; + + if (n_length || packed) /* Don't pack 0 length keys */ + { + uint next_length_pack, new_ref_length=s_temp->ref_length; + + if (packed) + { + /* If first key and next key is packed (only on delete) */ + if (!prev_key && org_key) + { + get_key_length(org_key_length,org_key); + key=start; + if (sort_order) /* SerG */ + { + while (key < end && sort_order[*key] == sort_order[*org_key]) + { + key++; org_key++; + } + } + else + { + while (key < end && *key == *org_key) + { + key++; org_key++; + } + } + if ((new_ref_length= (uint) (key - start))) + new_ref_length+=pack_marker; + } + + if (!n_length) + { + /* + We put a different key between two identical variable length keys + Extend next key to have same prefix as this key + */ + if (new_ref_length) /* prefix of previus key */ + { /* make next key longer */ + s_temp->part_of_prev_key= new_ref_length; + s_temp->prev_length= org_key_length - + (new_ref_length-pack_marker); + s_temp->n_ref_length= s_temp->part_of_prev_key; + s_temp->n_length= s_temp->prev_length; + n_length= get_pack_length(s_temp->prev_length); + s_temp->prev_key+= (new_ref_length - pack_marker); + length+= s_temp->prev_length + n_length; + } + else + { /* Can't use prev key */ + s_temp->part_of_prev_key=0; + s_temp->prev_length= org_key_length; + s_temp->n_ref_length=s_temp->n_length= org_key_length; + length+= org_key_length; + } + return (int) length; + } + + ref_length=n_length; + /* Get information about not packed key suffix */ + get_key_pack_length(n_length,next_length_pack,next_key); + + /* Test if new keys has fewer characters that match the previous key */ + if (!new_ref_length) + { /* Can't use prev key */ + s_temp->part_of_prev_key= 0; + s_temp->prev_length= ref_length; + s_temp->n_ref_length= s_temp->n_length= n_length+ref_length; + return (int) length+ref_length-next_length_pack; + } + if (ref_length+pack_marker > new_ref_length) + { + uint new_pack_length=new_ref_length-pack_marker; + /* We must copy characters from the original key to the next key */ + s_temp->part_of_prev_key= new_ref_length; + s_temp->prev_length= ref_length - new_pack_length; + s_temp->n_ref_length=s_temp->n_length=n_length + s_temp->prev_length; + s_temp->prev_key+= new_pack_length; + length-= (next_length_pack - get_pack_length(s_temp->n_length)); + return (int) length + s_temp->prev_length; + } + } + else + { + /* Next key wasn't a prefix of previous key */ + ref_length=0; + next_length_pack=0; + } + DBUG_PRINT("test",("length: %d next_key: %p", length, + next_key)); + + { + uint tmp_length; + key=(start+=ref_length); + if (key+n_length < key_end) /* Normalize length based */ + key_end=key+n_length; + if (sort_order) /* SerG */ + { + while (key < key_end && sort_order[*key] == + sort_order[*next_key]) + { + key++; next_key++; + } + } + else + { + while (key < key_end && *key == *next_key) + { + key++; next_key++; + } + } + if (!(tmp_length=(uint) (key-start))) + { /* Key can't be re-packed */ + s_temp->next_key_pos=0; + return length; + } + ref_length+=tmp_length; + n_length-=tmp_length; + length-=tmp_length+next_length_pack; /* We gained these chars */ + } + if (n_length == 0 && ref_length == new_key_length) + { + s_temp->n_ref_length=pack_marker; /* Same as prev key */ + } + else + { + s_temp->n_ref_length=ref_length | pack_marker; + length+= get_pack_length(n_length); + s_temp->n_length=n_length; + } + } + } + return length; +} + + +/* Length of key which is prefix compressed */ + +int +_mi_calc_bin_pack_key_length(MI_KEYDEF *keyinfo,uint nod_flag,uchar *next_key, + uchar *org_key, uchar *prev_key, uchar *key, + MI_KEY_PARAM *s_temp) +{ + uint length,key_length,ref_length; + + s_temp->totlength=key_length=_mi_keylength(keyinfo,key)+nod_flag; +#ifdef HAVE_valgrind + s_temp->n_length= s_temp->n_ref_length=0; /* For valgrind */ +#endif + s_temp->key=key; + s_temp->prev_key=org_key; + if (prev_key) /* If not first key in block */ + { + /* pack key against previous key */ + /* + As keys may be identical when running a sort in myisamchk, we + have to guard against the case where keys may be identical + */ + uchar *end; + end=key+key_length; + for ( ; *key == *prev_key && key < end; key++,prev_key++) ; + s_temp->ref_length= ref_length=(uint) (key-s_temp->key); + length=key_length - ref_length + get_pack_length(ref_length); + } + else + { + /* No previous key */ + s_temp->ref_length=ref_length=0; + length=key_length+1; + } + if ((s_temp->next_key_pos=next_key)) /* If another key after */ + { + /* pack key against next key */ + uint next_length,next_length_pack; + get_key_pack_length(next_length,next_length_pack,next_key); + + /* If first key and next key is packed (only on delete) */ + if (!prev_key && org_key && next_length) + { + uchar *end; + for (key= s_temp->key, end=key+next_length ; + *key == *org_key && key < end; + key++,org_key++) ; + ref_length= (uint) (key - s_temp->key); + } + + if (next_length > ref_length) + { + /* We put a key with different case between two keys with the same prefix + Extend next key to have same prefix as + this key */ + s_temp->n_ref_length= ref_length; + s_temp->prev_length= next_length-ref_length; + s_temp->prev_key+= ref_length; + return (int) (length+ s_temp->prev_length - next_length_pack + + get_pack_length(ref_length)); + } + /* Check how many characters are identical to next key */ + key= s_temp->key+next_length; + s_temp->prev_length= 0; + while (*key++ == *next_key++) ; + if ((ref_length= (uint) (key - s_temp->key)-1) == next_length) + { + s_temp->next_key_pos=0; + return length; /* can't pack next key */ + } + s_temp->n_ref_length=ref_length; + return (int) (length-(ref_length - next_length) - next_length_pack + + get_pack_length(ref_length)); + } + return (int) length; +} + + +/* +** store a key packed with _mi_calc_xxx_key_length in page-buffert +*/ + +/* store key without compression */ + +void _mi_store_static_key(MI_KEYDEF *keyinfo __attribute__((unused)), + register uchar *key_pos, + register MI_KEY_PARAM *s_temp) +{ + memcpy((uchar*) key_pos,(uchar*) s_temp->key,(size_t) s_temp->totlength); +} + + +/* store variable length key with prefix compression */ + +#define store_pack_length(test,pos,length) { \ + if (test) { *((pos)++) = (uchar) (length); } else \ + { *((pos)++) = (uchar) ((length) >> 8); *((pos)++) = (uchar) (length); } } + + +void _mi_store_var_pack_key(MI_KEYDEF *keyinfo __attribute__((unused)), + register uchar *key_pos, + register MI_KEY_PARAM *s_temp) +{ + uint length; + uchar *start; + + start=key_pos; + + if (s_temp->ref_length) + { + /* Packed against previous key */ + store_pack_length(s_temp->pack_marker == 128,key_pos,s_temp->ref_length); + /* If not same key after */ + if (s_temp->ref_length != s_temp->pack_marker) + store_key_length_inc(key_pos,s_temp->key_length); + } + else + { + /* Not packed against previous key */ + store_pack_length(s_temp->pack_marker == 128,key_pos,s_temp->key_length); + } + bmove((uchar*) key_pos,(uchar*) s_temp->key, + (length=s_temp->totlength-(uint) (key_pos-start))); + + if (!s_temp->next_key_pos) /* No following key */ + return; + key_pos+=length; + + if (s_temp->prev_length) + { + /* Extend next key because new key didn't have same prefix as prev key */ + if (s_temp->part_of_prev_key) + { + store_pack_length(s_temp->pack_marker == 128,key_pos, + s_temp->part_of_prev_key); + store_key_length_inc(key_pos,s_temp->n_length); + } + else + { + s_temp->n_length+= s_temp->store_not_null; + store_pack_length(s_temp->pack_marker == 128,key_pos, + s_temp->n_length); + } + memcpy(key_pos, s_temp->prev_key, s_temp->prev_length); + } + else if (s_temp->n_ref_length) + { + store_pack_length(s_temp->pack_marker == 128,key_pos,s_temp->n_ref_length); + if (s_temp->n_ref_length == s_temp->pack_marker) + return; /* Identical key */ + store_key_length(key_pos,s_temp->n_length); + } + else if (s_temp->n_length) + { + s_temp->n_length+= s_temp->store_not_null; + store_pack_length(s_temp->pack_marker == 128,key_pos,s_temp->n_length); + } +} + + +/* variable length key with prefix compression */ + +void _mi_store_bin_pack_key(MI_KEYDEF *keyinfo __attribute__((unused)), + register uchar *key_pos, + register MI_KEY_PARAM *s_temp) +{ + store_key_length_inc(key_pos,s_temp->ref_length); + memcpy((char*) key_pos,(char*) s_temp->key+s_temp->ref_length, + (size_t) s_temp->totlength-s_temp->ref_length); + + if (s_temp->next_key_pos) + { + key_pos+=(uint) (s_temp->totlength-s_temp->ref_length); + store_key_length_inc(key_pos,s_temp->n_ref_length); + if (s_temp->prev_length) /* If we must extend key */ + { + memcpy(key_pos,s_temp->prev_key,s_temp->prev_length); + } + } +} diff --git a/storage/myisam/mi_static.c b/storage/myisam/mi_static.c new file mode 100644 index 00000000..d0c3995d --- /dev/null +++ b/storage/myisam/mi_static.c @@ -0,0 +1,180 @@ +/* Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* + Static variables for MyISAM library. All definied here for easy making of + a shared library +*/ + +#ifndef MY_GLOBAL_INCLUDED +#include "myisamdef.h" +#endif + +LIST *myisam_open_list=0; +uchar myisam_file_magic[]= +{ (uchar) 254, (uchar) 254,'\007', '\001', }; +uchar myisam_pack_file_magic[]= +{ (uchar) 254, (uchar) 254,'\010', '\002', }; +char * myisam_log_filename=(char*) "myisam.log"; +File myisam_log_file= -1; +uint myisam_quick_table_bits=9; +ulong myisam_block_size= MI_KEY_BLOCK_LENGTH; /* Best by test */ +my_bool myisam_flush=0, myisam_delay_key_write=0, myisam_single_user=0; +#if !defined(DONT_USE_RW_LOCKS) +ulong myisam_concurrent_insert= 2; +#else +ulong myisam_concurrent_insert= 0; +#endif +ulonglong myisam_max_temp_length= MAX_FILE_SIZE; +ulong myisam_data_pointer_size=4; +ulonglong myisam_mmap_size= SIZE_T_MAX, myisam_mmap_used= 0; +my_bool (*mi_killed)(MI_INFO *)= mi_killed_standalone; + +/* + read_vec[] is used for converting between P_READ_KEY.. and SEARCH_ + Position is , == , >= , <= , > , < +*/ + +uint myisam_read_vec[]= +{ + SEARCH_FIND, SEARCH_FIND | SEARCH_BIGGER, SEARCH_FIND | SEARCH_SMALLER, + SEARCH_NO_FIND | SEARCH_BIGGER, SEARCH_NO_FIND | SEARCH_SMALLER, + SEARCH_FIND | SEARCH_PREFIX, SEARCH_LAST, SEARCH_LAST | SEARCH_SMALLER, + MBR_CONTAIN, MBR_INTERSECT, MBR_WITHIN, MBR_DISJOINT, MBR_EQUAL +}; + +uint myisam_readnext_vec[]= +{ + SEARCH_BIGGER, SEARCH_BIGGER, SEARCH_SMALLER, SEARCH_BIGGER, SEARCH_SMALLER, + SEARCH_BIGGER, SEARCH_SMALLER, SEARCH_SMALLER +}; + +PSI_memory_key mi_key_memory_MYISAM_SHARE; +PSI_memory_key mi_key_memory_MI_INFO; +PSI_memory_key mi_key_memory_MI_INFO_ft1_to_ft2; +PSI_memory_key mi_key_memory_MI_INFO_bulk_insert; +PSI_memory_key mi_key_memory_record_buffer; +PSI_memory_key mi_key_memory_FTB; +PSI_memory_key mi_key_memory_FT_INFO; +PSI_memory_key mi_key_memory_FTPARSER_PARAM; +PSI_memory_key mi_key_memory_ft_memroot; +PSI_memory_key mi_key_memory_ft_stopwords; +PSI_memory_key mi_key_memory_MI_SORT_PARAM; +PSI_memory_key mi_key_memory_MI_SORT_PARAM_wordroot; +PSI_memory_key mi_key_memory_SORT_FT_BUF; +PSI_memory_key mi_key_memory_SORT_KEY_BLOCKS; +PSI_memory_key mi_key_memory_filecopy; +PSI_memory_key mi_key_memory_SORT_INFO_buffer; +PSI_memory_key mi_key_memory_MI_DECODE_TREE; +PSI_memory_key mi_key_memory_MYISAM_SHARE_decode_tables; +PSI_memory_key mi_key_memory_preload_buffer; +PSI_memory_key mi_key_memory_stPageList_pages; +PSI_memory_key mi_key_memory_keycache_thread_var; + +#ifdef HAVE_PSI_INTERFACE +PSI_mutex_key mi_key_mutex_MYISAM_SHARE_intern_lock, + mi_key_mutex_MI_SORT_INFO_mutex, mi_key_mutex_MI_CHECK_print_msg; + +static PSI_mutex_info all_myisam_mutexes[]= +{ + { &mi_key_mutex_MI_SORT_INFO_mutex, "MI_SORT_INFO::mutex", 0}, + { &mi_key_mutex_MYISAM_SHARE_intern_lock, "MYISAM_SHARE::intern_lock", 0}, + { &mi_key_mutex_MI_CHECK_print_msg, "MI_CHECK::print_msg", 0} +}; + +PSI_rwlock_key mi_key_rwlock_MYISAM_SHARE_key_root_lock, + mi_key_rwlock_MYISAM_SHARE_mmap_lock; + +static PSI_rwlock_info all_myisam_rwlocks[]= +{ + { &mi_key_rwlock_MYISAM_SHARE_key_root_lock, "MYISAM_SHARE::key_root_lock", 0}, + { &mi_key_rwlock_MYISAM_SHARE_mmap_lock, "MYISAM_SHARE::mmap_lock", 0} +}; + +PSI_cond_key mi_key_cond_MI_SORT_INFO_cond; + +static PSI_cond_info all_myisam_conds[]= +{ + { &mi_key_cond_MI_SORT_INFO_cond, "MI_SORT_INFO::cond", 0} +}; + +PSI_file_key mi_key_file_datatmp, mi_key_file_dfile, mi_key_file_kfile, + mi_key_file_log; + +static PSI_file_info all_myisam_files[]= +{ + { & mi_key_file_datatmp, "data_tmp", 0}, + { & mi_key_file_dfile, "dfile", 0}, + { & mi_key_file_kfile, "kfile", 0}, + { & mi_key_file_log, "log", 0} +}; + +PSI_thread_key mi_key_thread_find_all_keys; + +static PSI_thread_info all_myisam_threads[]= +{ + { &mi_key_thread_find_all_keys, "find_all_keys", 0}, +}; + +static PSI_memory_info all_myisam_memory[]= +{ + { &mi_key_memory_MYISAM_SHARE, "MYISAM_SHARE", 0}, + { &mi_key_memory_MI_INFO, "MI_INFO", 0}, + { &mi_key_memory_MI_INFO_ft1_to_ft2, "MI_INFO::ft1_to_ft2", 0}, + { &mi_key_memory_MI_INFO_bulk_insert, "MI_INFO::bulk_insert", 0}, + { &mi_key_memory_record_buffer, "record_buffer", 0}, + { &mi_key_memory_FTB, "FTB", 0}, + { &mi_key_memory_FT_INFO, "FT_INFO", 0}, + { &mi_key_memory_FTPARSER_PARAM, "FTPARSER_PARAM", 0}, + { &mi_key_memory_ft_memroot, "ft_memroot", 0}, + { &mi_key_memory_ft_stopwords, "ft_stopwords", 0}, + { &mi_key_memory_MI_SORT_PARAM, "MI_SORT_PARAM", 0}, + { &mi_key_memory_MI_SORT_PARAM_wordroot, "MI_SORT_PARAM::wordroot", 0}, + { &mi_key_memory_SORT_FT_BUF, "SORT_FT_BUF", 0}, + { &mi_key_memory_SORT_KEY_BLOCKS, "SORT_KEY_BLOCKS", 0}, + { &mi_key_memory_filecopy, "filecopy", 0}, + { &mi_key_memory_SORT_INFO_buffer, "SORT_INFO::buffer", 0}, + { &mi_key_memory_MI_DECODE_TREE, "MI_DECODE_TREE", 0}, + { &mi_key_memory_MYISAM_SHARE_decode_tables, "MYISAM_SHARE::decode_tables", 0}, + { &mi_key_memory_preload_buffer, "preload_buffer", 0}, + { &mi_key_memory_stPageList_pages, "stPageList::pages", 0}, + { &mi_key_memory_keycache_thread_var, "keycache_thread_var", 0} +}; + +void init_myisam_psi_keys() +{ + const char* category= "myisam"; + int count; + + count= array_elements(all_myisam_mutexes); + mysql_mutex_register(category, all_myisam_mutexes, count); + + count= array_elements(all_myisam_rwlocks); + mysql_rwlock_register(category, all_myisam_rwlocks, count); + + count= array_elements(all_myisam_conds); + mysql_cond_register(category, all_myisam_conds, count); + + count= array_elements(all_myisam_files); + mysql_file_register(category, all_myisam_files, count); + + count= array_elements(all_myisam_threads); + mysql_thread_register(category, all_myisam_threads, count); + + count= array_elements(all_myisam_memory); + mysql_memory_register(category, all_myisam_memory, count); +} +#endif /* HAVE_PSI_INTERFACE */ + diff --git a/storage/myisam/mi_statrec.c b/storage/myisam/mi_statrec.c new file mode 100644 index 00000000..363c9707 --- /dev/null +++ b/storage/myisam/mi_statrec.c @@ -0,0 +1,311 @@ +/* Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + + /* Functions to handle fixed-length-records */ + +#include "myisamdef.h" + + +int _mi_write_static_record(MI_INFO *info, const uchar *record) +{ + uchar temp[8]; /* max pointer length */ + if (info->s->state.dellink != HA_OFFSET_ERROR && + !info->append_insert_at_end) + { + my_off_t filepos=info->s->state.dellink; + info->rec_cache.seek_not_done=1; /* We have done a seek */ + if (info->s->file_read(info, &temp[0],info->s->base.rec_reflength, + info->s->state.dellink+1, + MYF(MY_NABP))) + goto err; + info->s->state.dellink= _mi_rec_pos(info->s,temp); + info->state->del--; + info->state->empty-=info->s->base.pack_reclength; + if (info->s->file_write(info, record, info->s->base.reclength, + filepos, + MYF(MY_NABP))) + goto err; + } + else + { + if (info->state->data_file_length > info->s->base.max_data_file_length- + info->s->base.pack_reclength) + { + my_errno=HA_ERR_RECORD_FILE_FULL; + return(2); + } + if (info->opt_flag & WRITE_CACHE_USED) + { /* Cash in use */ + if (my_b_write(&info->rec_cache, record, + info->s->base.reclength)) + goto err; + if (info->s->base.pack_reclength != info->s->base.reclength) + { + uint length=info->s->base.pack_reclength - info->s->base.reclength; + bzero(temp,length); + if (my_b_write(&info->rec_cache, temp,length)) + goto err; + } + } + else + { + info->rec_cache.seek_not_done=1; /* We have done a seek */ + if (info->s->file_write(info, record, info->s->base.reclength, + info->state->data_file_length, + info->s->write_flag)) + goto err; + if (info->s->base.pack_reclength != info->s->base.reclength) + { + uint length=info->s->base.pack_reclength - info->s->base.reclength; + bzero(temp,length); + if (info->s->file_write(info, temp,length, + info->state->data_file_length+ + info->s->base.reclength, + info->s->write_flag)) + goto err; + } + } + info->state->data_file_length+=info->s->base.pack_reclength; + info->s->state.split++; + } + return 0; + err: + return 1; +} + +int _mi_update_static_record(MI_INFO *info, my_off_t pos, const uchar *record) +{ + info->rec_cache.seek_not_done=1; /* We have done a seek */ + return (info->s->file_write(info, + record, info->s->base.reclength, + pos, + MYF(MY_NABP)) != 0); +} + + +int _mi_delete_static_record(MI_INFO *info) +{ + uchar temp[9]; /* 1+sizeof(uint32) */ + + info->state->del++; + info->state->empty+=info->s->base.pack_reclength; + temp[0]= '\0'; /* Mark that record is deleted */ + _mi_dpointer(info,temp+1,info->s->state.dellink); + info->s->state.dellink = info->lastpos; + info->rec_cache.seek_not_done=1; + return (info->s->file_write(info,(uchar*) temp, 1+info->s->rec_reflength, + info->lastpos, MYF(MY_NABP)) != 0); +} + + +int _mi_cmp_static_record(register MI_INFO *info, register const uchar *old) +{ + DBUG_ENTER("_mi_cmp_static_record"); + + if (info->opt_flag & WRITE_CACHE_USED) + { + if (flush_io_cache(&info->rec_cache)) + { + DBUG_RETURN(-1); + } + info->rec_cache.seek_not_done=1; /* We have done a seek */ + } + + if ((info->opt_flag & READ_CHECK_USED)) + { /* If check isn't disabled */ + info->rec_cache.seek_not_done=1; /* We have done a seek */ + if (info->s->file_read(info, info->rec_buff, info->s->base.reclength, + info->lastpos, + MYF(MY_NABP))) + DBUG_RETURN(-1); + if (memcmp(info->rec_buff, old, + (uint) info->s->base.reclength)) + { + DBUG_DUMP("read",old,info->s->base.reclength); + DBUG_DUMP("disk",info->rec_buff,info->s->base.reclength); + my_errno=HA_ERR_RECORD_CHANGED; /* Record have changed */ + DBUG_RETURN(1); + } + } + DBUG_RETURN(0); +} + + +int _mi_cmp_static_unique(MI_INFO *info, MI_UNIQUEDEF *def, + const uchar *record, my_off_t pos) +{ + DBUG_ENTER("_mi_cmp_static_unique"); + + info->rec_cache.seek_not_done=1; /* We have done a seek */ + if (info->s->file_read(info, info->rec_buff, info->s->base.reclength, + pos, MYF(MY_NABP))) + DBUG_RETURN(-1); + DBUG_RETURN(mi_unique_comp(def, record, info->rec_buff, + def->null_are_equal)); +} + + + /* Read a fixed-length-record */ + /* Returns 0 if Ok. */ + /* 1 if record is deleted */ + /* MY_FILE_ERROR on read-error or locking-error */ + +int _mi_read_static_record(register MI_INFO *info, register my_off_t pos, + register uchar *record) +{ + int error; + + if (pos != HA_OFFSET_ERROR) + { + if (info->opt_flag & WRITE_CACHE_USED && + info->rec_cache.pos_in_file <= pos && + flush_io_cache(&info->rec_cache)) + return(-1); + info->rec_cache.seek_not_done=1; /* We have done a seek */ + + error=info->s->file_read(info, record, info->s->base.reclength, + pos,MYF(MY_NABP)) != 0; + fast_mi_writeinfo(info); + if (! error) + { + if (!*record) + { + my_errno=HA_ERR_RECORD_DELETED; + return(1); /* Record is deleted */ + } + info->update|= HA_STATE_AKTIV; /* Record is read */ + return(0); + } + return(-1); /* Error on read */ + } + fast_mi_writeinfo(info); /* No such record */ + return(-1); +} + + + +int _mi_read_rnd_static_record(MI_INFO *info, uchar *buf, + register my_off_t filepos, + my_bool skip_deleted_blocks) +{ + int locked,error,cache_read; + uint cache_length; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("_mi_read_rnd_static_record"); + + cache_read=0; + cache_length=0; + if (info->opt_flag & WRITE_CACHE_USED && + (info->rec_cache.pos_in_file <= filepos || skip_deleted_blocks) && + flush_io_cache(&info->rec_cache)) + DBUG_RETURN(my_errno); + if (info->opt_flag & READ_CACHE_USED) + { /* Cache in use */ + if (filepos == my_b_tell(&info->rec_cache) && + (skip_deleted_blocks || !filepos)) + { + cache_read=1; /* Read record using cache */ + cache_length=(uint) (info->rec_cache.read_end - info->rec_cache.read_pos); + } + else + info->rec_cache.seek_not_done=1; /* Filepos is changed */ + } + locked=0; + if (info->lock_type == F_UNLCK) + { + if (filepos >= info->state->data_file_length) + { /* Test if new records */ + if (_mi_readinfo(info,F_RDLCK,0)) + DBUG_RETURN(my_errno); + locked=1; + } + else + { /* We don't nead new info */ +#ifndef UNSAFE_LOCKING + if ((! cache_read || share->base.reclength > cache_length) && + share->tot_locks == 0) + { /* record not in cache */ + if (my_lock(share->kfile,F_RDLCK,0L,F_TO_EOF, + MYF(MY_SEEK_NOT_DONE) | info->lock_wait)) + DBUG_RETURN(my_errno); + locked=1; + } +#else + info->tmp_lock_type=F_RDLCK; +#endif + } + } + if (filepos >= info->state->data_file_length) + { + DBUG_PRINT("test",("filepos: %ld (%ld) records: %ld del: %ld", + (long) filepos/share->base.reclength, (long) filepos, + (long) info->state->records, (long) info->state->del)); + fast_mi_writeinfo(info); + DBUG_RETURN(my_errno=HA_ERR_END_OF_FILE); + } + info->lastpos= filepos; + info->nextpos= filepos+share->base.pack_reclength; + + if (! cache_read) /* No cacheing */ + { + if ((error=_mi_read_static_record(info,filepos,buf))) + { + if (error > 0) + error=my_errno=HA_ERR_RECORD_DELETED; + else + error=my_errno; + } + DBUG_RETURN(error); + } + + /* + Read record with caching. If my_b_read() returns TRUE, less than the + requested bytes have been read. In this case rec_cache.error is + either -1 for a read error, or contains the number of bytes copied + into the buffer. + */ + error=my_b_read(&info->rec_cache,(uchar*) buf,share->base.reclength); + if (info->s->base.pack_reclength != info->s->base.reclength && !error) + { + char tmp[8]; /* Skill fill bytes */ + error=my_b_read(&info->rec_cache,(uchar*) tmp, + info->s->base.pack_reclength - info->s->base.reclength); + } + if (locked) + (void) _mi_writeinfo(info,0); /* Unlock keyfile */ + if (!error) + { + if (!buf[0]) + { /* Record is removed */ + DBUG_RETURN(my_errno=HA_ERR_RECORD_DELETED); + } + /* Found and may be updated */ + info->update|= HA_STATE_AKTIV | HA_STATE_KEY_CHANGED; + DBUG_RETURN(0); + } + /* error is TRUE. my_errno should be set if rec_cache.error == -1 */ + if (info->rec_cache.error != -1 || my_errno == 0) + { + /* + If we could not get a full record, we either have a broken record, + or are at end of file. + */ + if (info->rec_cache.error == 0) + my_errno= HA_ERR_END_OF_FILE; + else + my_errno= HA_ERR_WRONG_IN_RECORD; + } + DBUG_RETURN(my_errno); /* Something wrong (EOF?) */ +} diff --git a/storage/myisam/mi_test1.c b/storage/myisam/mi_test1.c new file mode 100644 index 00000000..5a614edb --- /dev/null +++ b/storage/myisam/mi_test1.c @@ -0,0 +1,688 @@ +/* + Copyright (c) 2000, 2011, Oracle and/or its affiliates + Copyright (c) 2020, MariaDB Corporation. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Testing of the basic functions of a MyISAM table */ + +#include <my_global.h> +#include "myisam.h" +#include <my_getopt.h> +#include <m_string.h> + +#define MAX_REC_LENGTH 1024 + +static void usage(); + +static int rec_pointer_size=0, flags[50]; +static int key_field=FIELD_SKIP_PRESPACE,extra_field=FIELD_SKIP_ENDSPACE; +static int key_type=HA_KEYTYPE_NUM; +static int create_flag=0; + +static uint insert_count, update_count, remove_count; +static uint pack_keys=0, pack_seg=0, key_length; +static uint unique_key=HA_NOSAME; +static my_bool key_cacheing, null_fields, silent, skip_update, opt_unique, + verbose; +static MI_COLUMNDEF recinfo[4]; +static MI_KEYDEF keyinfo[10]; +static HA_KEYSEG keyseg[10]; +static HA_KEYSEG uniqueseg[10]; + +static int run_test(const char *filename); +static void get_options(int argc, char *argv[]); +static void create_key(uchar *key,uint rownr); +static void create_record(uchar *record,uint rownr); +static void update_record(uchar *record); + +int main(int argc,char *argv[]) +{ + MY_INIT(argv[0]); + my_init(); + if (key_cacheing) + init_key_cache(dflt_key_cache,KEY_CACHE_BLOCK_SIZE,IO_SIZE*16,0,0, + 0, DEFAULT_KEY_CACHE_PARTITIONS); + get_options(argc,argv); + + exit(run_test("test1")); +} + + +static int run_test(const char *filename) +{ + MI_INFO *file; + int i,j,error,deleted,rec_length,uniques=0; + ha_rows found,row_count; + my_off_t pos; + uchar record[MAX_REC_LENGTH],key[MAX_REC_LENGTH],read_record[MAX_REC_LENGTH]; + MI_UNIQUEDEF uniquedef; + MI_CREATE_INFO create_info; + + bzero((char*) recinfo,sizeof(recinfo)); + + /* First define 2 columns */ + recinfo[0].type=FIELD_NORMAL; recinfo[0].length=1; /* For NULL bits */ + recinfo[1].type=key_field; + recinfo[1].length= (key_field == FIELD_BLOB ? 4+portable_sizeof_char_ptr : + key_length); + if (key_field == FIELD_VARCHAR) + recinfo[1].length+= HA_VARCHAR_PACKLENGTH(key_length);; + recinfo[2].type=extra_field; + recinfo[2].length= (extra_field == FIELD_BLOB ? 4 + portable_sizeof_char_ptr : 24); + if (extra_field == FIELD_VARCHAR) + recinfo[2].length+= HA_VARCHAR_PACKLENGTH(recinfo[2].length); + recinfo[1].null_bit= null_fields ? 2 : 0; + + if (opt_unique) + { + recinfo[3].type=FIELD_CHECK; + recinfo[3].length=MI_UNIQUE_HASH_LENGTH; + } + rec_length=recinfo[0].length+recinfo[1].length+recinfo[2].length+ + recinfo[3].length; + + if (key_type == HA_KEYTYPE_VARTEXT1 && + key_length > 255) + key_type= HA_KEYTYPE_VARTEXT2; + + /* Define a key over the first column */ + keyinfo[0].seg=keyseg; + keyinfo[0].keysegs=1; + keyinfo[0].block_length= 0; /* Default block length */ + keyinfo[0].key_alg=HA_KEY_ALG_BTREE; + keyinfo[0].seg[0].type= key_type; + keyinfo[0].seg[0].flag= pack_seg; + keyinfo[0].seg[0].start=1; + keyinfo[0].seg[0].length=key_length; + keyinfo[0].seg[0].null_bit= null_fields ? 2 : 0; + keyinfo[0].seg[0].null_pos=0; + keyinfo[0].seg[0].language= default_charset_info->number; + if (pack_seg & HA_BLOB_PART) + { + keyinfo[0].seg[0].bit_start=4; /* Length of blob length */ + } + keyinfo[0].flag = (uint8) (pack_keys | unique_key); + + bzero((uchar*) flags,sizeof(flags)); + if (opt_unique) + { + uint start; + uniques=1; + bzero((char*) &uniquedef,sizeof(uniquedef)); + bzero((char*) uniqueseg,sizeof(uniqueseg)); + uniquedef.seg=uniqueseg; + uniquedef.keysegs=2; + + /* Make a unique over all columns (except first NULL fields) */ + for (i=0, start=1 ; i < 2 ; i++) + { + uniqueseg[i].start=start; + start+=recinfo[i+1].length; + uniqueseg[i].length=recinfo[i+1].length; + uniqueseg[i].language= default_charset_info->number; + } + uniqueseg[0].type= key_type; + uniqueseg[0].null_bit= null_fields ? 2 : 0; + uniqueseg[1].type= HA_KEYTYPE_TEXT; + if (extra_field == FIELD_BLOB) + { + uniqueseg[1].length=0; /* The whole blob */ + uniqueseg[1].bit_start=4; /* long blob */ + uniqueseg[1].flag|= HA_BLOB_PART; + } + else if (extra_field == FIELD_VARCHAR) + uniqueseg[1].flag|= HA_VAR_LENGTH_PART; + } + else + uniques=0; + + if (!silent) + printf("- Creating isam-file\n"); + bzero((char*) &create_info,sizeof(create_info)); + create_info.max_rows=(ulong) (rec_pointer_size ? + (1L << (rec_pointer_size*8))/40 : + 0); + if (mi_create(filename,1,keyinfo,3+opt_unique,recinfo, + uniques, &uniquedef, &create_info, + create_flag)) + goto err; + if (!(file=mi_open(filename,2,HA_OPEN_ABORT_IF_LOCKED))) + goto err; + if (!silent) + printf("- Writing key:s\n"); + + my_errno=0; + row_count=deleted=0; + for (i=49 ; i>=1 ; i-=2 ) + { + if (insert_count-- == 0) { (void) mi_close(file); exit(0) ; } + j=i%25 +1; + create_record(record,j); + error=mi_write(file,record); + if (!error) + row_count++; + flags[j]=1; + if (verbose || error) + printf("J= %2d mi_write: %d errno: %d\n", j,error,my_errno); + } + + /* Insert 2 rows with null values */ + if (null_fields) + { + create_record(record,0); + error=mi_write(file,record); + if (!error) + row_count++; + if (verbose || error) + printf("J= NULL mi_write: %d errno: %d\n", error,my_errno); + error=mi_write(file,record); + if (!error) + row_count++; + if (verbose || error) + printf("J= NULL mi_write: %d errno: %d\n", error,my_errno); + flags[0]=2; + } + + if (!skip_update) + { + if (opt_unique) + { + if (!silent) + printf("- Checking unique constraint\n"); + create_record(record,j); + if (!mi_write(file,record) || my_errno != HA_ERR_FOUND_DUPP_UNIQUE) + { + printf("unique check failed\n"); + } + } + if (!silent) + printf("- Updating rows\n"); + + /* Update first last row to force extend of file */ + if (mi_rsame(file,read_record,-1)) + { + printf("Can't find last row with mi_rsame\n"); + } + else + { + memcpy(record,read_record,rec_length); + update_record(record); + if (mi_update(file,read_record,record)) + { + printf("Can't update last row: %.*s\n", + keyinfo[0].seg[0].length,read_record+1); + } + } + + /* Read through all rows and update them */ + pos=(my_off_t) 0; + found=0; + while ((error=mi_rrnd(file,read_record,pos)) == 0) + { + if (update_count-- == 0) { (void) mi_close(file); exit(0) ; } + memcpy(record,read_record,rec_length); + update_record(record); + if (mi_update(file,read_record,record)) + { + printf("Can't update row: %.*s, error: %d\n", + keyinfo[0].seg[0].length,record+1,my_errno); + } + found++; + pos=HA_OFFSET_ERROR; + } + if (found != row_count) + printf("Found %ld of %ld rows\n", (ulong) found, (ulong) row_count); + } + + if (!silent) + printf("- Reopening file\n"); + if (mi_close(file)) goto err; + if (!(file=mi_open(filename,2,HA_OPEN_ABORT_IF_LOCKED))) goto err; + if (!skip_update) + { + if (!silent) + printf("- Removing keys\n"); + + for (i=0 ; i <= 10 ; i++) + { + /* testing */ + if (remove_count-- == 0) { (void) mi_close(file); exit(0) ; } + j=i*2; + if (!flags[j]) + continue; + create_key(key,j); + my_errno=0; + if ((error = mi_rkey(file,read_record,0,key,HA_WHOLE_KEY, + HA_READ_KEY_EXACT))) + { + if (verbose || (flags[j] >= 1 || + (error && my_errno != HA_ERR_KEY_NOT_FOUND))) + printf("key: '%.*s' mi_rkey: %3d errno: %3d\n", + (int) key_length, key + MY_TEST(null_fields), error, my_errno); + } + else + { + error=mi_delete(file,read_record); + if (verbose || error) + printf("key: '%.*s' mi_delete: %3d errno: %3d\n", + (int) key_length, key + MY_TEST(null_fields), error, my_errno); + if (! error) + { + deleted++; + flags[j]--; + } + } + } + } + if (!silent) + printf("- Reading rows with key\n"); + for (i=0 ; i <= 25 ; i++) + { + create_key(key,i); + my_errno=0; + error=mi_rkey(file,read_record,0,key,HA_WHOLE_KEY,HA_READ_KEY_EXACT); + if (verbose || + (error == 0 && flags[i] == 0 && unique_key) || + (error && (flags[i] != 0 || my_errno != HA_ERR_KEY_NOT_FOUND))) + { + printf("key: '%.*s' mi_rkey: %3d errno: %3d record: %s\n", + (int) key_length, key + MY_TEST(null_fields), error, my_errno, + record + 1); + } + } + + if (!silent) + printf("- Reading rows with position\n"); + for (i=1,found=0 ; i <= 30 ; i++) + { + my_errno=0; + if ((error=mi_rrnd(file,read_record,i == 1 ? 0L : HA_OFFSET_ERROR)) == -1) + { + if (found != row_count-deleted) + printf("Found only %ld of %ld rows\n", (ulong) found, + (ulong) (row_count - deleted)); + break; + } + if (!error) + found++; + if (verbose || (error != 0 && error != HA_ERR_RECORD_DELETED && + error != HA_ERR_END_OF_FILE)) + { + printf("pos: %2d mi_rrnd: %3d errno: %3d record: %s\n", + i-1,error,my_errno,read_record+1); + } + } + if (mi_close(file)) goto err; + my_end(MY_CHECK_ERROR); + + return (0); +err: + printf("got error: %3d when using myisam-database\n",my_errno); + return 1; /* skip warning */ +} + + +static void create_key_part(uchar *key,uint rownr) +{ + if (!unique_key) + rownr&=7; /* Some identical keys */ + if (keyinfo[0].seg[0].type == HA_KEYTYPE_NUM) + { + sprintf((char*) key,"%*d",keyinfo[0].seg[0].length,rownr); + } + else if (keyinfo[0].seg[0].type == HA_KEYTYPE_VARTEXT1 || + keyinfo[0].seg[0].type == HA_KEYTYPE_VARTEXT2) + { /* Alpha record */ + /* Create a key that may be easily packed */ + bfill(key,keyinfo[0].seg[0].length,rownr < 10 ? 'A' : 'B'); + sprintf((char*) key+keyinfo[0].seg[0].length-2,"%-2d",rownr); + if ((rownr & 7) == 0) + { + /* Change the key to force a unpack of the next key */ + bfill(key+3,keyinfo[0].seg[0].length-4,rownr < 10 ? 'a' : 'b'); + } + } + else + { /* Alpha record */ + if (keyinfo[0].seg[0].flag & HA_SPACE_PACK) + sprintf((char*) key,"%-*d",keyinfo[0].seg[0].length,rownr); + else + { + /* Create a key that may be easily packed */ + bfill(key,keyinfo[0].seg[0].length,rownr < 10 ? 'A' : 'B'); + sprintf((char*) key+keyinfo[0].seg[0].length-2,"%-2d",rownr); + if ((rownr & 7) == 0) + { + /* Change the key to force a unpack of the next key */ + key[1]= (rownr < 10 ? 'a' : 'b'); + } + } + } +} + + +static void create_key(uchar *key,uint rownr) +{ + if (keyinfo[0].seg[0].null_bit) + { + if (rownr == 0) + { + key[0]=1; /* null key */ + key[1]=0; /* Fore easy print of key */ + return; + } + *key++=0; + } + if (keyinfo[0].seg[0].flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART)) + { + size_t tmp; + create_key_part(key+2,rownr); + tmp=strlen((char*) key+2); + int2store(key,tmp); + } + else + create_key_part(key,rownr); +} + + +static uchar blob_key[MAX_REC_LENGTH]; +static uchar blob_record[MAX_REC_LENGTH+20*20]; + + +static void create_record(uchar *record,uint rownr) +{ + uchar *pos; + bzero((char*) record,MAX_REC_LENGTH); + record[0]=1; /* delete marker */ + if (rownr == 0 && keyinfo[0].seg[0].null_bit) + record[0]|=keyinfo[0].seg[0].null_bit; /* Null key */ + + pos=record+1; + if (recinfo[1].type == FIELD_BLOB) + { + size_t tmp; + uchar *ptr; + create_key_part(blob_key,rownr); + tmp=strlen((char*) blob_key); + int4store(pos,tmp); + ptr=blob_key; + memcpy(pos+4, &ptr, sizeof(char*)); + pos+=recinfo[1].length; + } + else if (recinfo[1].type == FIELD_VARCHAR) + { + size_t tmp, pack_length= HA_VARCHAR_PACKLENGTH(recinfo[1].length-1); + create_key_part(pos+pack_length,rownr); + tmp= strlen((char*) pos+pack_length); + if (pack_length == 1) + *(uchar*) pos= (uchar) tmp; + else + int2store(pos,tmp); + pos+= recinfo[1].length; + } + else + { + create_key_part(pos,rownr); + pos+=recinfo[1].length; + } + if (recinfo[2].type == FIELD_BLOB) + { + size_t tmp; + uchar *ptr;; + sprintf((char*) blob_record,"... row: %d", rownr); + strappend((char*) blob_record,MY_MAX(MAX_REC_LENGTH-rownr,10),' '); + tmp=strlen((char*) blob_record); + int4store(pos,tmp); + ptr=blob_record; + memcpy(pos+4, &ptr, sizeof(char*)); + } + else if (recinfo[2].type == FIELD_VARCHAR) + { + size_t tmp, pack_length= HA_VARCHAR_PACKLENGTH(recinfo[1].length-1); + sprintf((char*) pos+pack_length, "... row: %d", rownr); + tmp= strlen((char*) pos+pack_length); + if (pack_length == 1) + *pos= (uchar) tmp; + else + int2store(pos,tmp); + } + else + { + sprintf((char*) pos,"... row: %d", rownr); + strappend((char*) pos,recinfo[2].length,' '); + } +} + +/* change row to test re-packing of rows and reallocation of keys */ + +static void update_record(uchar *record) +{ + uchar *pos=record+1; + if (recinfo[1].type == FIELD_BLOB) + { + uchar *column,*ptr; + int length; + length=uint4korr(pos); /* Long blob */ + memcpy(&column, pos+4, sizeof(char*)); + memcpy(blob_key,column,length); /* Move old key */ + ptr=blob_key; + memcpy(pos+4, &ptr, sizeof(char*)); /* Store pointer to new key */ + if (keyinfo[0].seg[0].type != HA_KEYTYPE_NUM) + my_ci_casedn(default_charset_info, (char*) blob_key, length, + (char*) blob_key, length); + pos+=recinfo[1].length; + } + else if (recinfo[1].type == FIELD_VARCHAR) + { + uint pack_length= HA_VARCHAR_PACKLENGTH(recinfo[1].length-1); + uint length= pack_length == 1 ? (uint) *(uchar*) pos : uint2korr(pos); + my_ci_casedn(default_charset_info, (char*) pos + pack_length, length, + (char*) pos + pack_length, length); + pos+=recinfo[1].length; + } + else + { + if (keyinfo[0].seg[0].type != HA_KEYTYPE_NUM) + my_ci_casedn(default_charset_info, (char*) pos, keyinfo[0].seg[0].length, + (char*) pos, keyinfo[0].seg[0].length); + pos+=recinfo[1].length; + } + + if (recinfo[2].type == FIELD_BLOB) + { + uchar *column; + int length; + length=uint4korr(pos); + memcpy(&column, pos+4, sizeof(char*)); + memcpy(blob_record,column,length); + bfill(blob_record+length,20,'.'); /* Make it larger */ + length+=20; + int4store(pos,length); + column= blob_record; + memcpy(pos+4, &column, sizeof(char*)); + } + else if (recinfo[2].type == FIELD_VARCHAR) + { + /* Second field is longer than 10 characters */ + uint pack_length= HA_VARCHAR_PACKLENGTH(recinfo[1].length-1); + uint length= pack_length == 1 ? (uint) *(uchar*) pos : uint2korr(pos); + bfill(pos+pack_length+length,recinfo[2].length-length-pack_length,'.'); + length=recinfo[2].length-pack_length; + if (pack_length == 1) + *(uchar*) pos= (uchar) length; + else + int2store(pos,length); + } + else + { + bfill(pos+recinfo[2].length-10,10,'.'); + } +} + + +static struct my_option my_long_options[] = +{ + {"checksum", 'c', "Undocumented", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, +#ifndef DBUG_OFF + {"debug", '#', "Undocumented", + 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, +#endif + {"delete_rows", 'd', "Undocumented", &remove_count, + &remove_count, 0, GET_UINT, REQUIRED_ARG, 1000, 0, 0, 0, 0, 0}, + {"help", '?', "Display help and exit", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"insert_rows", 'i', "Undocumented", &insert_count, + &insert_count, 0, GET_UINT, REQUIRED_ARG, 1000, 0, 0, 0, 0, 0}, + {"key_alpha", 'a', "Use a key of type HA_KEYTYPE_TEXT", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"key_binary_pack", 'B', "Undocumented", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"key_blob", 'b', "Undocumented", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"key_cache", 'K', "Undocumented", &key_cacheing, + &key_cacheing, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"key_length", 'k', "Undocumented", &key_length, &key_length, + 0, GET_UINT, REQUIRED_ARG, 6, 0, 0, 0, 0, 0}, + {"key_multiple", 'm', "Undocumented", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"key_prefix_pack", 'P', "Undocumented", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"key_space_pack", 'p', "Undocumented", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"key_varchar", 'w', "Test VARCHAR keys", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"null_fields", 'N', "Define fields with NULL", + &null_fields, &null_fields, 0, GET_BOOL, NO_ARG, + 0, 0, 0, 0, 0, 0}, + {"row_fixed_size", 'S', "Undocumented", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"row_pointer_size", 'R', "Undocumented", &rec_pointer_size, + &rec_pointer_size, 0, GET_INT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"silent", 's', "Undocumented", + &silent, &silent, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"skip_update", 'U', "Undocumented", &skip_update, + &skip_update, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"unique", 'C', "Undocumented", &opt_unique, &opt_unique, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"update_rows", 'u', "Undocumented", &update_count, + &update_count, 0, GET_UINT, REQUIRED_ARG, 1000, 0, 0, 0, 0, 0}, + {"verbose", 'v', "Be more verbose", &verbose, &verbose, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"version", 'V', "Print version number and exit", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + + +static my_bool +get_one_option(const struct my_option *opt, + const char *argument __attribute__((unused)), + const char *filename __attribute__((unused))) +{ + switch(opt->id) { + case 'a': + key_type= HA_KEYTYPE_TEXT; + break; + case 'c': + create_flag|= HA_CREATE_CHECKSUM; + break; + case 'R': /* Length of record pointer */ + if (rec_pointer_size > 3) + rec_pointer_size=0; + break; + case 'P': + pack_keys= HA_PACK_KEY; /* Use prefix compression */ + break; + case 'B': + pack_keys= HA_BINARY_PACK_KEY; /* Use binary compression */ + break; + case 'S': + if (key_field == FIELD_VARCHAR) + { + create_flag=0; /* Static sized varchar */ + } + else if (key_field != FIELD_BLOB) + { + key_field=FIELD_NORMAL; /* static-size record */ + extra_field=FIELD_NORMAL; + } + break; + case 'p': + pack_keys=HA_PACK_KEY; /* Use prefix + space packing */ + pack_seg=HA_SPACE_PACK; + key_type=HA_KEYTYPE_TEXT; + break; + case 'm': + unique_key=0; + break; + case 'b': + key_field=FIELD_BLOB; /* blob key */ + extra_field= FIELD_BLOB; + pack_seg|= HA_BLOB_PART; + key_type= HA_KEYTYPE_VARTEXT1; + break; + case 'k': + if (key_length < 4 || key_length > HA_MAX_KEY_LENGTH) + { + fprintf(stderr,"Wrong key length\n"); + exit(1); + } + break; + case 'w': + key_field=FIELD_VARCHAR; /* varchar keys */ + extra_field= FIELD_VARCHAR; + key_type= HA_KEYTYPE_VARTEXT1; + pack_seg|= HA_VAR_LENGTH_PART; + create_flag|= HA_PACK_RECORD; + break; + case 'K': /* Use key cacheing */ + key_cacheing=1; + break; + case 'V': + printf("test1 Ver 1.2 \n"); + exit(0); + case '#': + DBUG_PUSH (argument); + break; + case '?': + usage(); + exit(1); + } + return 0; +} + + +/* Read options */ + +static void get_options(int argc, char *argv[]) +{ + int ho_error; + + if ((ho_error=handle_options(&argc, &argv, my_long_options, get_one_option))) + exit(ho_error); + + return; +} /* get options */ + + +static void usage() +{ + printf("Usage: %s [options]\n\n", my_progname); + my_print_help(my_long_options); + my_print_variables(my_long_options); +} + +#include "mi_extrafunc.h" diff --git a/storage/myisam/mi_test2.c b/storage/myisam/mi_test2.c new file mode 100644 index 00000000..4b5039ee --- /dev/null +++ b/storage/myisam/mi_test2.c @@ -0,0 +1,1059 @@ +/* + Copyright (c) 2000, 2010, Oracle and/or its affiliates + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Test av isam-databas: stor test */ + +#ifndef USE_MY_FUNC /* We want to be able to dbug this !! */ +#define USE_MY_FUNC +#endif +#include "myisamdef.h" +#include <m_ctype.h> +#include <my_bit.h> + +#define STANDARD_LENGTH 37 +#define MYISAM_KEYS 6 +#define MAX_PARTS 4 + +static void get_options(int argc, char *argv[]); +static uint rnd(uint max_value); +static void fix_length(uchar *record,uint length); +static void put_blob_in_record(uchar *blob_pos,char **blob_buffer); +static void copy_key(struct st_myisam_info *info,uint inx, + uchar *record,uchar *key); + +static int verbose=0,testflag=0, + first_key=0,async_io=0,key_cacheing=0,write_cacheing=0,do_locking=0, + rec_pointer_size=0,pack_fields=1,use_log=0,silent=0, + opt_quick_mode=0; +static int pack_seg=HA_SPACE_PACK,pack_type=HA_PACK_KEY,remove_count=-1, + create_flag=0; +static ulong key_cache_size=IO_SIZE*16; +static uint key_cache_block_size= KEY_CACHE_BLOCK_SIZE; + +static uint keys=MYISAM_KEYS,recant=1000; +static uint use_blob=0; +static uint16 key1[1001],key3[5000]; +static uchar record[300],record2[300],key[100],key2[100]; +static uchar read_record[300],read_record2[300],read_record3[300]; +static HA_KEYSEG glob_keyseg[MYISAM_KEYS][MAX_PARTS]; + + /* Test program */ + +int main(int argc, char *argv[]) +{ + uint i; + int j,n1,n2,n3,error,k; + uint write_count,update,dupp_keys,opt_delete,start,length,blob_pos, + reclength,ant,found_parts; + my_off_t lastpos; + ha_rows range_records,records; + MI_INFO *file; + MI_KEYDEF keyinfo[10]; + MI_COLUMNDEF recinfo[10]; + MI_ISAMINFO info; + const char *filename; + char *blob_buffer; + MI_CREATE_INFO create_info; + page_range pages; + MY_INIT(argv[0]); + + filename= "test2"; + get_options(argc,argv); + if (! async_io) + my_disable_async_io=1; + + reclength=STANDARD_LENGTH+60+(use_blob ? 8 : 0); + blob_pos=STANDARD_LENGTH+60; + keyinfo[0].seg= &glob_keyseg[0][0]; + keyinfo[0].seg[0].start=0; + keyinfo[0].seg[0].length=6; + keyinfo[0].seg[0].type=HA_KEYTYPE_TEXT; + keyinfo[0].seg[0].language= default_charset_info->number; + keyinfo[0].seg[0].flag=(uint8) pack_seg; + keyinfo[0].seg[0].null_bit=0; + keyinfo[0].seg[0].null_pos=0; + keyinfo[0].key_alg=HA_KEY_ALG_BTREE; + keyinfo[0].keysegs=1; + keyinfo[0].flag = pack_type; + keyinfo[0].block_length= 0; /* Default block length */ + keyinfo[1].seg= &glob_keyseg[1][0]; + keyinfo[1].seg[0].start=7; + keyinfo[1].seg[0].length=6; + keyinfo[1].seg[0].type=HA_KEYTYPE_BINARY; + keyinfo[1].seg[0].flag=0; + keyinfo[1].seg[0].null_bit=0; + keyinfo[1].seg[0].null_pos=0; + keyinfo[1].seg[1].start=0; /* two part key */ + keyinfo[1].seg[1].length=6; + keyinfo[1].seg[1].type=HA_KEYTYPE_NUM; + keyinfo[1].seg[1].flag=HA_REVERSE_SORT; + keyinfo[1].seg[1].null_bit=0; + keyinfo[1].seg[1].null_pos=0; + keyinfo[1].key_alg=HA_KEY_ALG_BTREE; + keyinfo[1].keysegs=2; + keyinfo[1].flag =0; + keyinfo[1].block_length= MI_MIN_KEY_BLOCK_LENGTH; /* Diff blocklength */ + keyinfo[2].seg= &glob_keyseg[2][0]; + keyinfo[2].seg[0].start=12; + keyinfo[2].seg[0].length=8; + keyinfo[2].seg[0].type=HA_KEYTYPE_BINARY; + keyinfo[2].seg[0].flag=HA_REVERSE_SORT; + keyinfo[2].seg[0].null_bit=0; + keyinfo[2].seg[0].null_pos=0; + keyinfo[2].key_alg=HA_KEY_ALG_BTREE; + keyinfo[2].keysegs=1; + keyinfo[2].flag =HA_NOSAME; + keyinfo[2].block_length= 0; /* Default block length */ + keyinfo[3].seg= &glob_keyseg[3][0]; + keyinfo[3].seg[0].start=0; + keyinfo[3].seg[0].length=reclength-(use_blob ? 8 : 0); + keyinfo[3].seg[0].type=HA_KEYTYPE_TEXT; + keyinfo[3].seg[0].language=default_charset_info->number; + keyinfo[3].seg[0].flag=(uint8) pack_seg; + keyinfo[3].seg[0].null_bit=0; + keyinfo[3].seg[0].null_pos=0; + keyinfo[3].key_alg=HA_KEY_ALG_BTREE; + keyinfo[3].keysegs=1; + keyinfo[3].flag = pack_type; + keyinfo[3].block_length= 0; /* Default block length */ + keyinfo[4].seg= &glob_keyseg[4][0]; + keyinfo[4].seg[0].start=0; + keyinfo[4].seg[0].length=5; + keyinfo[4].seg[0].type=HA_KEYTYPE_TEXT; + keyinfo[4].seg[0].language=default_charset_info->number; + keyinfo[4].seg[0].flag=0; + keyinfo[4].seg[0].null_bit=0; + keyinfo[4].seg[0].null_pos=0; + keyinfo[4].key_alg=HA_KEY_ALG_BTREE; + keyinfo[4].keysegs=1; + keyinfo[4].flag = pack_type; + keyinfo[4].block_length= 0; /* Default block length */ + keyinfo[5].seg= &glob_keyseg[5][0]; + keyinfo[5].seg[0].start=0; + keyinfo[5].seg[0].length=4; + keyinfo[5].seg[0].type=HA_KEYTYPE_TEXT; + keyinfo[5].seg[0].language=default_charset_info->number; + keyinfo[5].seg[0].flag=pack_seg; + keyinfo[5].seg[0].null_bit=0; + keyinfo[5].seg[0].null_pos=0; + keyinfo[5].key_alg=HA_KEY_ALG_BTREE; + keyinfo[5].keysegs=1; + keyinfo[5].flag = pack_type; + keyinfo[5].block_length= 0; /* Default block length */ + + recinfo[0].type=pack_fields ? FIELD_SKIP_PRESPACE : 0; + recinfo[0].length=7; + recinfo[0].null_bit=0; + recinfo[0].null_pos=0; + recinfo[1].type=pack_fields ? FIELD_SKIP_PRESPACE : 0; + recinfo[1].length=5; + recinfo[1].null_bit=0; + recinfo[1].null_pos=0; + recinfo[2].type=pack_fields ? FIELD_SKIP_PRESPACE : 0; + recinfo[2].length=9; + recinfo[2].null_bit=0; + recinfo[2].null_pos=0; + recinfo[3].type=FIELD_NORMAL; + recinfo[3].length=STANDARD_LENGTH-7-5-9-4; + recinfo[3].null_bit=0; + recinfo[3].null_pos=0; + recinfo[4].type=pack_fields ? FIELD_SKIP_ZERO : 0; + recinfo[4].length=4; + recinfo[4].null_bit=0; + recinfo[4].null_pos=0; + recinfo[5].type=pack_fields ? FIELD_SKIP_ENDSPACE : 0; + recinfo[5].length=60; + recinfo[5].null_bit=0; + recinfo[5].null_pos=0; + if (use_blob) + { + recinfo[6].type=FIELD_BLOB; + recinfo[6].length=4+portable_sizeof_char_ptr; + recinfo[6].null_bit=0; + recinfo[6].null_pos=0; + } + + write_count=update=dupp_keys=opt_delete=0; + blob_buffer=0; + + for (i=1000 ; i>0 ; i--) key1[i]=0; + for (i=4999 ; i>0 ; i--) key3[i]=0; + + if (!silent) + printf("- Creating isam-file\n"); + /* DBUG_PUSH(""); */ + /* my_delete(filename,MYF(0)); */ /* Remove old locks under gdb */ + file= 0; + bzero((char*) &create_info,sizeof(create_info)); + create_info.max_rows=(ha_rows) (rec_pointer_size ? + (1L << (rec_pointer_size*8))/ + reclength : 0); + create_info.reloc_rows=(ha_rows) 100; + if (mi_create(filename,keys,&keyinfo[first_key], + use_blob ? 7 : 6, &recinfo[0], + 0,(MI_UNIQUEDEF*) 0, + &create_info,create_flag)) + goto err; + if (use_log) + mi_log(1); + if (!(file=mi_open(filename,2,HA_OPEN_ABORT_IF_LOCKED))) + goto err; + if (!silent) + printf("- Writing key:s\n"); + if (key_cacheing) + init_key_cache(dflt_key_cache,key_cache_block_size,key_cache_size,0,0, + 0, DEFAULT_KEY_CACHE_PARTITIONS); + if (do_locking) + mi_lock_database(file,F_WRLCK); + if (write_cacheing) + mi_extra(file,HA_EXTRA_WRITE_CACHE,0); + if (opt_quick_mode) + mi_extra(file,HA_EXTRA_QUICK,0); + + for (i=0 ; i < recant ; i++) + { + n1=rnd(1000); n2=rnd(100); n3=rnd(5000); + sprintf((char*) record,"%6d:%4d:%8d:Pos: %4d ",n1,n2,n3,write_count); + int4store(record+STANDARD_LENGTH-4,(long) i); + fix_length(record,(uint) STANDARD_LENGTH+rnd(60)); + put_blob_in_record(record+blob_pos,&blob_buffer); + DBUG_PRINT("test",("record: %d",i)); + + if (mi_write(file,record)) + { + if (my_errno != HA_ERR_FOUND_DUPP_KEY || key3[n3] == 0) + { + printf("Error: %d in write at record: %d\n",my_errno,i); + goto err; + } + if (verbose) printf(" Double key: %d\n",n3); + } + else + { + if (key3[n3] == 1 && first_key <3 && first_key+keys >= 3) + { + printf("Error: Didn't get error when writing second key: '%8d'\n",n3); + goto err; + } + write_count++; key1[n1]++; key3[n3]=1; + } + + /* Check if we can find key without flushing database */ + if (i == recant/2) + { + for (j=rnd(1000)+1 ; j>0 && key1[j] == 0 ; j--) ; + if (!j) + for (j=999 ; j>0 && key1[j] == 0 ; j--) ; + sprintf((char*) key,"%6d",j); + if (mi_rkey(file,read_record,0,key,HA_WHOLE_KEY,HA_READ_KEY_EXACT)) + { + printf("Test in loop: Can't find key: \"%s\"\n",key); + goto err; + } + } + } + if (testflag==1) goto end; + + if (write_cacheing) + { + if (mi_extra(file,HA_EXTRA_NO_CACHE,0)) + { + puts("got error from mi_extra(HA_EXTRA_NO_CACHE)"); + goto end; + } + } + if (key_cacheing) + resize_key_cache(dflt_key_cache,key_cache_block_size,key_cache_size*2, + 0, 0, 0); + + if (!silent) + printf("- Delete\n"); + for (i=0 ; i<recant/10 ; i++) + { + for (j=rnd(1000)+1 ; j>0 && key1[j] == 0 ; j--) ; + if (j != 0) + { + sprintf((char*) key,"%6d",j); + if (mi_rkey(file,read_record,0,key,HA_WHOLE_KEY,HA_READ_KEY_EXACT)) + { + printf("can't find key1: \"%s\"\n",key); + goto err; + } + if (opt_delete == (uint) remove_count) /* While testing */ + goto end; + if (mi_delete(file,read_record)) + { + printf("error: %d; can't delete record: \"%s\"\n", my_errno,read_record); + goto err; + } + opt_delete++; + key1[atoi((char*) read_record+keyinfo[0].seg[0].start)]--; + key3[atoi((char*) read_record+keyinfo[2].seg[0].start)]=0; + } + else + puts("Warning: Skipping delete test because no dupplicate keys"); + } + if (testflag==2) goto end; + + if (!silent) + printf("- Update\n"); + for (i=0 ; i<recant/10 ; i++) + { + n1=rnd(1000); n2=rnd(100); n3=rnd(5000); + sprintf((char*) record2,"%6d:%4d:%8d:XXX: %4d ",n1,n2,n3,update); + int4store(record2+STANDARD_LENGTH-4,(long) i); + fix_length(record2,(uint) STANDARD_LENGTH+rnd(60)); + + for (j=rnd(1000)+1 ; j>0 && key1[j] == 0 ; j--) ; + if (j != 0) + { + sprintf((char*) key,"%6d",j); + if (mi_rkey(file,read_record,0,key,HA_WHOLE_KEY,HA_READ_KEY_EXACT)) + { + printf("can't find key1: \"%s\"\n",(char*) key); + goto err; + } + if (use_blob) + { + if (i & 1) + put_blob_in_record(record2+blob_pos,&blob_buffer); + else + bmove(record2+blob_pos,read_record+blob_pos,8); + } + if (mi_update(file,read_record,record2)) + { + if (my_errno != HA_ERR_FOUND_DUPP_KEY || key3[n3] == 0) + { + printf("error: %d; can't update:\nFrom: \"%s\"\nTo: \"%s\"\n", + my_errno,read_record,record2); + goto err; + } + if (verbose) + printf("Double key when tried to update:\nFrom: \"%s\"\nTo: \"%s\"\n",record,record2); + } + else + { + key1[atoi((char*) read_record+keyinfo[0].seg[0].start)]--; + key3[atoi((char*) read_record+keyinfo[2].seg[0].start)]=0; + key1[n1]++; key3[n3]=1; + update++; + } + } + } + if (testflag == 3) + goto end; + + for (i=999, dupp_keys=j=0 ; i>0 ; i--) + { + if (key1[i] > dupp_keys) + { + dupp_keys=key1[i]; j=i; + } + } + sprintf((char*) key,"%6d",j); + start=keyinfo[0].seg[0].start; + length=keyinfo[0].seg[0].length; + if (dupp_keys) + { + if (!silent) + printf("- Same key: first - next -> last - prev -> first\n"); + DBUG_PRINT("progpos",("first - next -> last - prev -> first")); + if (verbose) printf(" Using key: \"%s\" Keys: %d\n",key,dupp_keys); + + if (mi_rkey(file,read_record,0,key,HA_WHOLE_KEY,HA_READ_KEY_EXACT)) + goto err; + if (mi_rsame(file,read_record2,-1)) + goto err; + if (memcmp(read_record,read_record2,reclength) != 0) + { + printf("mi_rsame didn't find same record\n"); + goto end; + } + info.recpos=mi_position(file); + if (mi_rfirst(file,read_record2,0) || + mi_rsame_with_pos(file,read_record2,0,info.recpos) || + memcmp(read_record,read_record2,reclength) != 0) + { + printf("mi_rsame_with_pos didn't find same record\n"); + goto end; + } + { + int skr=mi_rnext(file,read_record2,0); + if ((skr && my_errno != HA_ERR_END_OF_FILE) || + mi_rprev(file,read_record2,-1) || + memcmp(read_record,read_record2,reclength) != 0) + { + printf("mi_rsame_with_pos lost position\n"); + goto end; + } + } + ant=1; + while (mi_rnext(file,read_record2,0) == 0 && + memcmp(read_record2+start,key,length) == 0) ant++; + if (ant != dupp_keys) + { + printf("next: Found: %d keys of %d\n",ant,dupp_keys); + goto end; + } + ant=0; + while (mi_rprev(file,read_record3,0) == 0 && + memcmp(read_record3+start,key,length) == 0) ant++; + if (ant != dupp_keys) + { + printf("prev: Found: %d records of %d\n",ant,dupp_keys); + goto end; + } + + /* Check of mi_rnext_same */ + if (mi_rkey(file,read_record,0,key,HA_WHOLE_KEY,HA_READ_KEY_EXACT)) + goto err; + ant=1; + while (!mi_rnext_same(file,read_record3) && ant < dupp_keys+10) + ant++; + if (ant != dupp_keys || my_errno != HA_ERR_END_OF_FILE) + { + printf("mi_rnext_same: Found: %d records of %d\n",ant,dupp_keys); + goto end; + } + } + + if (!silent) + printf("- All keys: first - next -> last - prev -> first\n"); + DBUG_PRINT("progpos",("All keys: first - next -> last - prev -> first")); + ant=1; + if (mi_rfirst(file,read_record,0)) + { + printf("Can't find first record\n"); + goto end; + } + while ((error=mi_rnext(file,read_record3,0)) == 0 && ant < write_count+10) + ant++; + if (ant != write_count - opt_delete || error != HA_ERR_END_OF_FILE) + { + printf("next: I found: %d records of %d (error: %d)\n", + ant, write_count - opt_delete, error); + goto end; + } + if (mi_rlast(file,read_record2,0) || + memcmp(read_record2,read_record3,reclength)) + { + printf("Can't find last record\n"); + DBUG_DUMP("record2",(uchar*) read_record2,reclength); + DBUG_DUMP("record3",(uchar*) read_record3,reclength); + goto end; + } + ant=1; + while (mi_rprev(file,read_record3,0) == 0 && ant < write_count+10) + ant++; + if (ant != write_count - opt_delete) + { + printf("prev: I found: %d records of %d\n",ant,write_count); + goto end; + } + if (memcmp(read_record,read_record3,reclength)) + { + printf("Can't find first record\n"); + goto end; + } + + if (!silent) + printf("- Test if: Read first - next - prev - prev - next == first\n"); + DBUG_PRINT("progpos",("- Read first - next - prev - prev - next == first")); + if (mi_rfirst(file,read_record,0) || + mi_rnext(file,read_record3,0) || + mi_rprev(file,read_record3,0) || + mi_rprev(file,read_record3,0) == 0 || + mi_rnext(file,read_record3,0)) + goto err; + if (memcmp(read_record,read_record3,reclength) != 0) + printf("Can't find first record\n"); + + if (!silent) + printf("- Test if: Read last - prev - next - next - prev == last\n"); + DBUG_PRINT("progpos",("Read last - prev - next - next - prev == last")); + if (mi_rlast(file,read_record2,0) || + mi_rprev(file,read_record3,0) || + mi_rnext(file,read_record3,0) || + mi_rnext(file,read_record3,0) == 0 || + mi_rprev(file,read_record3,0)) + goto err; + if (memcmp(read_record2,read_record3,reclength)) + printf("Can't find last record\n"); +#ifdef NOT_ANYMORE + if (!silent) + puts("- Test read key-part"); + strmov(key2,key); + for(i=strlen(key2) ; i-- > 1 ;) + { + key2[i]=0; + + /* The following row is just to catch some bugs in the key code */ + bzero((char*) file->lastkey,file->s->base.max_key_length*2); + if (mi_rkey(file,read_record,0,key2,(uint) i,HA_READ_PREFIX)) + goto err; + if (memcmp(read_record+start,key,(uint) i)) + { + puts("Didn't find right record"); + goto end; + } + } +#endif + if (dupp_keys > 2) + { + if (!silent) + printf("- Read key (first) - next - delete - next -> last\n"); + DBUG_PRINT("progpos",("first - next - delete - next -> last")); + if (mi_rkey(file,read_record,0,key,HA_WHOLE_KEY,HA_READ_KEY_EXACT)) + goto err; + if (mi_rnext(file,read_record3,0)) goto err; + if (mi_delete(file,read_record3)) goto err; + opt_delete++; + ant=1; + while (mi_rnext(file,read_record3,0) == 0 && + memcmp(read_record3+start,key,length) == 0) ant++; + if (ant != dupp_keys-1) + { + printf("next: I can only find: %d keys of %d\n",ant,dupp_keys-1); + goto end; + } + } + if (dupp_keys>4) + { + if (!silent) + printf("- Read last of key - prev - delete - prev -> first\n"); + DBUG_PRINT("progpos",("last - prev - delete - prev -> first")); + if (mi_rprev(file,read_record3,0)) goto err; + if (mi_rprev(file,read_record3,0)) goto err; + if (mi_delete(file,read_record3)) goto err; + opt_delete++; + ant=1; + while (mi_rprev(file,read_record3,0) == 0 && + memcmp(read_record3+start,key,length) == 0) ant++; + if (ant != dupp_keys-2) + { + printf("next: I can only find: %d keys of %d\n",ant,dupp_keys-2); + goto end; + } + } + if (dupp_keys > 6) + { + if (!silent) + printf("- Read first - delete - next -> last\n"); + DBUG_PRINT("progpos",("first - delete - next -> last")); + if (mi_rkey(file,read_record3,0,key,HA_WHOLE_KEY,HA_READ_KEY_EXACT)) + goto err; + if (mi_delete(file,read_record3)) goto err; + opt_delete++; + ant=1; + if (mi_rnext(file,read_record,0)) + goto err; /* Skall finnas poster */ + while (mi_rnext(file,read_record3,0) == 0 && + memcmp(read_record3+start,key,length) == 0) ant++; + if (ant != dupp_keys-3) + { + printf("next: I can only find: %d keys of %d\n",ant,dupp_keys-3); + goto end; + } + + if (!silent) + printf("- Read last - delete - prev -> first\n"); + DBUG_PRINT("progpos",("last - delete - prev -> first")); + if (mi_rprev(file,read_record3,0)) goto err; + if (mi_delete(file,read_record3)) goto err; + opt_delete++; + ant=0; + while (mi_rprev(file,read_record3,0) == 0 && + memcmp(read_record3+start,key,length) == 0) ant++; + if (ant != dupp_keys-4) + { + printf("next: I can only find: %d keys of %d\n",ant,dupp_keys-4); + goto end; + } + } + + if (!silent) + puts("- Test if: Read rrnd - same"); + DBUG_PRINT("progpos",("Read rrnd - same")); + for (i=0 ; i < write_count ; i++) + { + if (mi_rrnd(file,read_record,i == 0 ? 0L : HA_OFFSET_ERROR) == 0) + break; + } + if (i == write_count) + goto err; + + bmove(read_record2,read_record,reclength); + for (i=MY_MIN(2,keys) ; i-- > 0 ;) + { + if (mi_rsame(file,read_record2,(int) i)) goto err; + if (memcmp(read_record,read_record2,reclength) != 0) + { + printf("mi_rsame didn't find same record\n"); + goto end; + } + } + if (!silent) + puts("- Test mi_records_in_range"); + mi_status(file,&info,HA_STATUS_VARIABLE); + for (i=0 ; i < info.keys ; i++) + { + key_range min_key, max_key; + if (mi_rfirst(file,read_record,(int) i) || + mi_rlast(file,read_record2,(int) i)) + goto err; + copy_key(file,(uint) i,(uchar*) read_record,(uchar*) key); + copy_key(file,(uint) i,(uchar*) read_record2,(uchar*) key2); + min_key.key= key; + min_key.keypart_map= HA_WHOLE_KEY; + min_key.flag= HA_READ_KEY_EXACT; + max_key.key= key2; + max_key.keypart_map= HA_WHOLE_KEY; + max_key.flag= HA_READ_AFTER_KEY; + + range_records= mi_records_in_range(file,(int) i, &min_key, &max_key, + &pages); + if (range_records < info.records*8/10 || + range_records > info.records*12/10) + { + printf("mi_records_range returned %ld; Should be about %ld\n", + (long) range_records,(long) info.records); + goto end; + } + if (verbose) + { + printf("mi_records_range returned %ld; Exact is %ld (diff: %4.2g %%)\n", + (long) range_records, (long) info.records, + labs((long) range_records - (long) info.records)*100.0/ + info.records); + } + } + for (i=0 ; i < 5 ; i++) + { + for (j=rnd(1000)+1 ; j>0 && key1[j] == 0 ; j--) ; + for (k=rnd(1000)+1 ; k>0 && key1[k] == 0 ; k--) ; + if (j != 0 && k != 0) + { + key_range min_key, max_key; + page_range pages; + if (j > k) + swap_variables(int, j, k); + sprintf((char*) key,"%6d",j); + sprintf((char*) key2,"%6d",k); + + min_key.key= key; + min_key.keypart_map= HA_WHOLE_KEY; + min_key.flag= HA_READ_AFTER_KEY; + max_key.key= key2; + max_key.keypart_map= HA_WHOLE_KEY; + max_key.flag= HA_READ_BEFORE_KEY; + range_records= mi_records_in_range(file, 0, &min_key, &max_key, &pages); + records=0; + for (j++ ; j < k ; j++) + records+=key1[j]; + if ((long) range_records < (long) records*7/10-2 || + (long) range_records > (long) records*14/10+2) + { + printf("mi_records_range for key: %d returned %lu; Should be about %lu\n", + i, (ulong) range_records, (ulong) records); + goto end; + } + if (verbose && records) + { + printf("mi_records_range returned %lu; Exact is %lu (diff: %4.2g %%)\n", + (ulong) range_records, (ulong) records, + labs((long) range_records-(long) records)*100.0/records); + + } + } + } + + if (!silent) + printf("- mi_info\n"); + mi_status(file,&info,HA_STATUS_VARIABLE | HA_STATUS_CONST); + if (info.records != write_count-opt_delete || info.deleted > opt_delete + update + || info.keys != keys) + { + puts("Wrong info from mi_info"); + printf("Got: records: %lu delete: %lu i_keys: %d\n", + (ulong) info.records, (ulong) info.deleted, info.keys); + } + if (verbose) + { + char buff[80]; + get_date(buff,3,info.create_time); + printf("info: Created %s\n",buff); + get_date(buff,3,info.check_time); + printf("info: checked %s\n",buff); + get_date(buff,3,info.update_time); + printf("info: Modified %s\n",buff); + } + + mi_panic(HA_PANIC_WRITE); + mi_panic(HA_PANIC_READ); + if (mi_is_changed(file)) + puts("Warning: mi_is_changed reported that datafile was changed"); + + if (!silent) + printf("- mi_extra(CACHE) + mi_rrnd.... + mi_extra(NO_CACHE)\n"); + if (mi_reset(file) || mi_extra(file,HA_EXTRA_CACHE,0)) + { + if (do_locking || (!use_blob && !pack_fields)) + { + puts("got error from mi_extra(HA_EXTRA_CACHE)"); + goto end; + } + } + ant=0; + while ((error=mi_rrnd(file,record,HA_OFFSET_ERROR)) != HA_ERR_END_OF_FILE && + ant < write_count + 10) + ant+= error ? 0 : 1; + if (ant != write_count-opt_delete) + { + printf("rrnd with cache: I can only find: %d records of %d\n", + ant,write_count-opt_delete); + goto end; + } + if (mi_extra(file,HA_EXTRA_NO_CACHE,0)) + { + puts("got error from mi_extra(HA_EXTRA_NO_CACHE)"); + goto end; + } + + ant=0; + mi_scan_init(file); + while ((error=mi_scan(file,record)) != HA_ERR_END_OF_FILE && + ant < write_count + 10) + ant+= error ? 0 : 1; + if (ant != write_count-opt_delete) + { + printf("scan with cache: I can only find: %d records of %d\n", + ant,write_count-opt_delete); + goto end; + } + + if (testflag == 4) goto end; + + if (!silent) + printf("- Removing keys\n"); + DBUG_PRINT("progpos",("Removing keys")); + lastpos = HA_OFFSET_ERROR; + /* DBUG_POP(); */ + mi_reset(file); + found_parts=0; + while ((error=mi_rrnd(file,read_record,HA_OFFSET_ERROR)) != + HA_ERR_END_OF_FILE) + { + info.recpos=mi_position(file); + if (lastpos >= info.recpos && lastpos != HA_OFFSET_ERROR) + { + printf("mi_rrnd didn't advance filepointer; old: %ld, new: %ld\n", + (long) lastpos, (long) info.recpos); + goto err; + } + lastpos=info.recpos; + if (error == 0) + { + if (opt_delete == (uint) remove_count) /* While testing */ + goto end; + if (mi_rsame(file,read_record,-1)) + { + printf("can't find record %lx\n",(long) info.recpos); + goto err; + } + if (use_blob) + { + ulong blob_length,pos; + uchar *ptr; + memcpy(&ptr, read_record+blob_pos+4, sizeof(ptr)); + blob_length= uint4korr(read_record+blob_pos); + for (pos=0 ; pos < blob_length ; pos++) + { + if (ptr[pos] != (uchar) (blob_length+pos)) + { + printf("found blob with wrong info at %ld\n",(long) lastpos); + use_blob=0; + break; + } + } + } + if (mi_delete(file,read_record)) + { + printf("can't delete record: %6.6s, delete_count: %d\n", + read_record, opt_delete); + goto err; + } + opt_delete++; + } + else + found_parts++; + } + if (my_errno != HA_ERR_END_OF_FILE && my_errno != HA_ERR_RECORD_DELETED) + printf("error: %d from mi_rrnd\n",my_errno); + if (write_count != opt_delete) + { + printf("Deleted only %d of %d records (%d parts)\n",opt_delete,write_count, + found_parts); + goto err; + } +end: + if (mi_close(file)) + goto err; + mi_panic(HA_PANIC_CLOSE); /* Should close log */ + if (!silent) + { + KEY_CACHE_STATISTICS stats; + + printf("\nFollowing test have been made:\n"); + printf("Write records: %d\nUpdate records: %d\nSame-key-read: %d\nDelete records: %d\n", write_count,update,dupp_keys,opt_delete); + if (rec_pointer_size) + printf("Record pointer size: %d\n",rec_pointer_size); + printf("myisam_block_size: %lu\n", myisam_block_size); + if (key_cacheing) + { + puts("Key cache used"); + printf("key_cache_block_size: %u\n", key_cache_block_size); + if (write_cacheing) + puts("Key cache resized"); + } + if (write_cacheing) + puts("Write cacheing used"); + if (write_cacheing) + puts("quick mode"); + if (async_io && do_locking) + puts("Asyncron io with locking used"); + else if (do_locking) + puts("Locking used"); + if (use_blob) + puts("blobs used"); + bzero(&stats, sizeof(stats)); + get_key_cache_statistics(dflt_key_cache, 0, &stats); + printf("key cache status: \n\ +blocks used:%10lu\n\ +not flushed:%10lu\n\ +w_requests: %10lu\n\ +writes: %10lu\n\ +r_requests: %10lu\n\ +reads: %10lu\n", + (ulong) stats.blocks_used, + (ulong) stats.blocks_changed, + (ulong) stats.write_requests, + (ulong) stats.writes, + (ulong) stats.read_requests, + (ulong) stats.reads); + } + end_key_cache(dflt_key_cache,1); + if (blob_buffer) + my_free(blob_buffer); + my_end(silent ? MY_CHECK_ERROR : MY_CHECK_ERROR | MY_GIVE_INFO); + return(0); +err: + printf("got error: %d when using MyISAM-database\n",my_errno); + if (file) + (void) mi_close(file); + return(1); +} /* main */ + + + /* l{ser optioner */ + /* OBS! intierar endast DEBUG - ingen debuggning h{r ! */ + +static void get_options(int argc, char **argv) +{ + char *pos,*progname; + + progname= argv[0]; + + while (--argc >0 && *(pos = *(++argv)) == '-' ) { + switch(*++pos) { + case 'B': + pack_type= HA_BINARY_PACK_KEY; + break; + case 'b': + use_blob=1; + break; + case 'K': /* Use key cacheing */ + key_cacheing=1; + if (*++pos) + key_cache_size=atol(pos); + break; + case 'W': /* Use write cacheing */ + write_cacheing=1; + if (*++pos) + my_default_record_cache_size=atoi(pos); + break; + case 'd': + remove_count= atoi(++pos); + break; + case 'i': + if (*++pos) + srand(atoi(pos)); + break; + case 'l': + use_log=1; + break; + case 'L': + do_locking=1; + break; + case 'A': /* use asyncron io */ + async_io=1; + if (*++pos) + my_default_record_cache_size=atoi(pos); + break; + case 'v': /* verbose */ + verbose=1; + break; + case 'm': /* records */ + if ((recant=atoi(++pos)) < 10) + { + fprintf(stderr,"record count must be >= 10\n"); + exit(1); + } + break; + case 'e': /* myisam_block_length */ + if ((myisam_block_size= atoi(++pos)) < MI_MIN_KEY_BLOCK_LENGTH || + myisam_block_size > MI_MAX_KEY_BLOCK_LENGTH) + { + fprintf(stderr,"Wrong myisam_block_length\n"); + exit(1); + } + myisam_block_size= my_round_up_to_next_power(myisam_block_size); + break; + case 'E': /* myisam_block_length */ + if ((key_cache_block_size=atoi(++pos)) < MI_MIN_KEY_BLOCK_LENGTH || + key_cache_block_size > MI_MAX_KEY_BLOCK_LENGTH) + { + fprintf(stderr,"Wrong key_cache_block_size\n"); + exit(1); + } + key_cache_block_size= my_round_up_to_next_power(key_cache_block_size); + break; + case 'f': + if ((first_key=atoi(++pos)) < 0 || first_key >= MYISAM_KEYS) + first_key=0; + break; + case 'k': + if ((keys=(uint) atoi(++pos)) < 1 || + keys > (uint) (MYISAM_KEYS-first_key)) + keys=MYISAM_KEYS-first_key; + break; + case 'P': + pack_type=0; /* Don't use DIFF_LENGTH */ + pack_seg=0; + break; + case 'R': /* Length of record pointer */ + rec_pointer_size=atoi(++pos); + if (rec_pointer_size > 7) + rec_pointer_size=0; + break; + case 'S': + pack_fields=0; /* Static-length-records */ + break; + case 's': + silent=1; + break; + case 't': + testflag=atoi(++pos); /* testmod */ + break; + case 'q': + opt_quick_mode=1; + break; + case 'c': + create_flag|= HA_CREATE_CHECKSUM; + break; + case 'D': + create_flag|=HA_CREATE_DELAY_KEY_WRITE; + break; + case '?': + case 'I': + case 'V': + printf("%s Ver 1.2 for %s at %s\n",progname,SYSTEM_TYPE,MACHINE_TYPE); + puts("By Monty, for your professional use\n"); + printf("Usage: %s [-?AbBcDIKLPRqSsVWltv] [-k#] [-f#] [-m#] [-e#] [-E#] [-t#]\n", + progname); + exit(0); + case '#': + DBUG_PUSH (++pos); + break; + default: + printf("Illegal option: '%c'\n",*pos); + break; + } + } + return; +} /* get options */ + + /* Get a random value 0 <= x <= n */ + +static uint rnd(uint max_value) +{ + return (uint) ((rand() & 32767)/32767.0*max_value); +} /* rnd */ + + + /* Create a variable length record */ + +static void fix_length(uchar *rec, uint length) +{ + bmove(rec+STANDARD_LENGTH, + "0123456789012345678901234567890123456789012345678901234567890", + length-STANDARD_LENGTH); + strfill((char*) rec+length,STANDARD_LENGTH+60-length,' '); +} /* fix_length */ + + + /* Put maybe a blob in record */ + +static void put_blob_in_record(uchar *blob_pos, char **blob_buffer) +{ + ulong i,length; + if (use_blob) + { + if (rnd(10) == 0) + { + if (! *blob_buffer && + !(*blob_buffer=my_malloc(PSI_NOT_INSTRUMENTED, use_blob,MYF(MY_WME)))) + { + use_blob=0; + return; + } + length=rnd(use_blob); + for (i=0 ; i < length ; i++) + (*blob_buffer)[i]=(char) (length+i); + int4store(blob_pos,length); + memcpy(blob_pos+4, blob_buffer, sizeof(char*)); + } + else + { + int4store(blob_pos,0); + } + } + return; +} + + +static void copy_key(MI_INFO *info,uint inx,uchar *rec,uchar *key_buff) +{ + HA_KEYSEG *keyseg; + + for (keyseg=info->s->keyinfo[inx].seg ; keyseg->type ; keyseg++) + { + memcpy(key_buff,rec+keyseg->start,(size_t) keyseg->length); + key_buff+=keyseg->length; + } + return; +} + +#include "mi_extrafunc.h" diff --git a/storage/myisam/mi_test3.c b/storage/myisam/mi_test3.c new file mode 100644 index 00000000..74684edd --- /dev/null +++ b/storage/myisam/mi_test3.c @@ -0,0 +1,499 @@ +/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Test av locking */ + +#ifndef _WIN32 /*no fork() in Windows*/ + +#include <my_global.h> +#include "myisam.h" +#include <sys/types.h> +#ifdef HAVE_SYS_WAIT_H +# include <sys/wait.h> +#endif +#ifndef WEXITSTATUS +# define WEXITSTATUS(stat_val) ((unsigned)(stat_val) >> 8) +#endif +#ifndef WIFEXITED +# define WIFEXITED(stat_val) (((stat_val) & 255) == 0) +#endif + + +#if defined(HAVE_LRAND48) +#define rnd(X) (lrand48() % X) +#define rnd_init(X) srand48(X) +#else +#define rnd(X) (random() % X) +#define rnd_init(X) srandom(X) +#endif + + +const char *filename= "test3"; +uint tests=10,forks=10,key_cacheing=0,use_log=0; + +static void get_options(int argc, char *argv[]); +void start_test(int id); +int test_read(MI_INFO *,int),test_write(MI_INFO *,int,int), + test_update(MI_INFO *,int,int),test_rrnd(MI_INFO *,int); + +struct record { + uchar id[8]; + uchar nr[4]; + uchar text[10]; +} record; + + +int main(int argc,char **argv) +{ + int status,wait_ret; + uint i=0; + MI_KEYDEF keyinfo[10]; + MI_COLUMNDEF recinfo[10]; + HA_KEYSEG keyseg[10][2]; + MY_INIT(argv[0]); + get_options(argc,argv); + + bzero((char*) keyinfo,sizeof(keyinfo)); + bzero((char*) recinfo,sizeof(recinfo)); + bzero((char*) keyseg,sizeof(keyseg)); + keyinfo[0].seg= &keyseg[0][0]; + keyinfo[0].seg[0].start=0; + keyinfo[0].seg[0].length=8; + keyinfo[0].seg[0].type=HA_KEYTYPE_TEXT; + keyinfo[0].seg[0].flag=HA_SPACE_PACK; + keyinfo[0].key_alg=HA_KEY_ALG_BTREE; + keyinfo[0].keysegs=1; + keyinfo[0].flag = (uint8) HA_PACK_KEY; + keyinfo[0].block_length= 0; /* Default block length */ + keyinfo[1].seg= &keyseg[1][0]; + keyinfo[1].seg[0].start=8; + keyinfo[1].seg[0].length=4; /* Long is always 4 in myisam */ + keyinfo[1].seg[0].type=HA_KEYTYPE_LONG_INT; + keyinfo[1].seg[0].flag=0; + keyinfo[1].key_alg=HA_KEY_ALG_BTREE; + keyinfo[1].keysegs=1; + keyinfo[1].flag =HA_NOSAME; + keyinfo[1].block_length= 0; /* Default block length */ + + recinfo[0].type=0; + recinfo[0].length=sizeof(record.id); + recinfo[1].type=0; + recinfo[1].length=sizeof(record.nr); + recinfo[2].type=0; + recinfo[2].length=sizeof(record.text); + + puts("- Creating myisam-file"); + my_delete(filename,MYF(0)); /* Remove old locks under gdb */ + if (mi_create(filename,2,&keyinfo[0],2,&recinfo[0],0,(MI_UNIQUEDEF*) 0, + (MI_CREATE_INFO*) 0,0)) + exit(1); + + rnd_init(0); + printf("- Starting %d processes\n",forks); fflush(stdout); + for (i=0 ; i < forks; i++) + { + if (!fork()) + { + start_test(i+1); + sleep(1); + return 0; + } + (void) rnd(1); + } + + for (i=0 ; i < forks ; i++) + while ((wait_ret=wait(&status)) && wait_ret == -1); + return 0; +} + + +static void get_options(int argc, char **argv) +{ + char *pos,*progname; + + progname= argv[0]; + + while (--argc >0 && *(pos = *(++argv)) == '-' ) { + switch(*++pos) { + case 'l': + use_log=1; + break; + case 'f': + forks=atoi(++pos); + break; + case 't': + tests=atoi(++pos); + break; + case 'K': /* Use key cacheing */ + key_cacheing=1; + break; + case 'A': /* All flags */ + use_log=key_cacheing=1; + break; + case '?': + case 'I': + case 'V': + printf("%s Ver 1.0 for %s at %s\n",progname,SYSTEM_TYPE,MACHINE_TYPE); + puts("By Monty, for your professional use\n"); + puts("Test av locking with threads\n"); + printf("Usage: %s [-?lKA] [-f#] [-t#]\n",progname); + exit(0); + case '#': + DBUG_PUSH (++pos); + break; + default: + printf("Illegal option: '%c'\n",*pos); + break; + } + } + return; +} + + +void start_test(int id) +{ + uint i; + int error,lock_type; + MI_ISAMINFO isam_info; + MI_INFO *file,*file1,*file2=0,*lock; + + if (use_log) + mi_log(1); + if (!(file1=mi_open(filename,O_RDWR,HA_OPEN_WAIT_IF_LOCKED)) || + !(file2=mi_open(filename,O_RDWR,HA_OPEN_WAIT_IF_LOCKED))) + { + fprintf(stderr,"Can't open isam-file: %s\n",filename); + exit(1); + } + if (key_cacheing && rnd(2) == 0) + init_key_cache(dflt_key_cache, KEY_CACHE_BLOCK_SIZE, 65536L, 0, 0, + 0, DEFAULT_KEY_CACHE_PARTITIONS); + printf("Process %d, pid: %ld\n", id, (long) getpid()); + fflush(stdout); + + for (error=i=0 ; i < tests && !error; i++) + { + file= (rnd(2) == 1) ? file1 : file2; + lock=0 ; lock_type=0; + if (rnd(10) == 0) + { + if (mi_lock_database(lock=(rnd(2) ? file1 : file2), + lock_type=(rnd(2) == 0 ? F_RDLCK : F_WRLCK))) + { + fprintf(stderr,"%2d: start: Can't lock table %d\n",id,my_errno); + error=1; + break; + } + } + switch (rnd(4)) { + case 0: error=test_read(file,id); break; + case 1: error=test_rrnd(file,id); break; + case 2: error=test_write(file,id,lock_type); break; + case 3: error=test_update(file,id,lock_type); break; + } + if (lock) + mi_lock_database(lock,F_UNLCK); + } + if (!error) + { + mi_status(file1,&isam_info,HA_STATUS_VARIABLE); + printf("%2d: End of test. Records: %ld Deleted: %ld\n", + id,(long) isam_info.records, (long) isam_info.deleted); + fflush(stdout); + } + + mi_close(file1); + mi_close(file2); + if (use_log) + mi_log(0); + if (error) + { + printf("%2d: Aborted\n",id); fflush(stdout); + exit(1); + } +} + + +int test_read(MI_INFO *file,int id) +{ + uint i,lock,found,next,prev; + ulong find; + + lock=0; + if (rnd(2) == 0) + { + lock=1; + if (mi_lock_database(file,F_RDLCK)) + { + fprintf(stderr,"%2d: Can't lock table %d\n",id,my_errno); + return 1; + } + } + + found=next=prev=0; + for (i=0 ; i < 100 ; i++) + { + find=rnd(100000); + if (!mi_rkey(file,record.id,1,(uchar*) &find, HA_WHOLE_KEY, + HA_READ_KEY_EXACT)) + found++; + else + { + if (my_errno != HA_ERR_KEY_NOT_FOUND) + { + fprintf(stderr,"%2d: Got error %d from read in read\n",id,my_errno); + return 1; + } + else if (!mi_rnext(file,record.id,1)) + next++; + else + { + if (my_errno != HA_ERR_END_OF_FILE) + { + fprintf(stderr,"%2d: Got error %d from rnext in read\n",id,my_errno); + return 1; + } + else if (!mi_rprev(file,record.id,1)) + prev++; + else + { + if (my_errno != HA_ERR_END_OF_FILE) + { + fprintf(stderr,"%2d: Got error %d from rnext in read\n", + id,my_errno); + return 1; + } + } + } + } + } + if (lock) + { + if (mi_lock_database(file,F_UNLCK)) + { + fprintf(stderr,"%2d: Can't unlock table\n",id); + return 1; + } + } + printf("%2d: read: found: %5d next: %5d prev: %5d\n", + id,found,next,prev); + fflush(stdout); + return 0; +} + + +int test_rrnd(MI_INFO *file,int id) +{ + uint count,lock; + + lock=0; + if (rnd(2) == 0) + { + lock=1; + if (mi_lock_database(file,F_RDLCK)) + { + fprintf(stderr,"%2d: Can't lock table (%d)\n",id,my_errno); + mi_close(file); + return 1; + } + if (rnd(2) == 0) + mi_extra(file,HA_EXTRA_CACHE,0); + } + + count=0; + if (mi_rrnd(file,record.id,0L)) + { + if (my_errno == HA_ERR_END_OF_FILE) + goto end; + fprintf(stderr,"%2d: Can't read first record (%d)\n",id,my_errno); + return 1; + } + for (count=1 ; !mi_rrnd(file,record.id,HA_OFFSET_ERROR) ;count++) ; + if (my_errno != HA_ERR_END_OF_FILE) + { + fprintf(stderr,"%2d: Got error %d from rrnd\n",id,my_errno); + return 1; + } + +end: + if (lock) + { + mi_extra(file,HA_EXTRA_NO_CACHE,0); + if (mi_lock_database(file,F_UNLCK)) + { + fprintf(stderr,"%2d: Can't unlock table\n",id); + exit(0); + } + } + printf("%2d: rrnd: %5d\n",id,count); fflush(stdout); + return 0; +} + + +int test_write(MI_INFO *file,int id,int lock_type) +{ + uint i,tries,count,lock; + + lock=0; + if (rnd(2) == 0 || lock_type == F_RDLCK) + { + lock=1; + if (mi_lock_database(file,F_WRLCK)) + { + if (lock_type == F_RDLCK && my_errno == EDEADLK) + { + printf("%2d: write: deadlock\n",id); fflush(stdout); + return 0; + } + fprintf(stderr,"%2d: Can't lock table (%d)\n",id,my_errno); + mi_close(file); + return 1; + } + if (rnd(2) == 0) + mi_extra(file,HA_EXTRA_WRITE_CACHE,0); + } + + my_snprintf((char*) record.id, sizeof(record.id), "%7ld", (long) getpid()); + strnmov((char*) record.text,"Testing...", sizeof(record.text)); + + tries=(uint) rnd(100)+10; + for (i=count=0 ; i < tries ; i++) + { + uint32 tmp=rnd(80000)+20000; + int4store(record.nr,tmp); + if (!mi_write(file,record.id)) + count++; + else + { + if (my_errno != HA_ERR_FOUND_DUPP_KEY) + { + fprintf(stderr,"%2d: Got error %d (errno %d) from write\n",id,my_errno, + errno); + return 1; + } + } + } + if (lock) + { + mi_extra(file,HA_EXTRA_NO_CACHE,0); + if (mi_lock_database(file,F_UNLCK)) + { + fprintf(stderr,"%2d: Can't unlock table\n",id); + exit(0); + } + } + printf("%2d: write: %5d\n",id,count); fflush(stdout); + return 0; +} + + +int test_update(MI_INFO *file,int id,int lock_type) +{ + uint i,lock,update; + uint32 tmp; + char find[4]; + struct record new_record; + + lock=0; + if (rnd(2) == 0 || lock_type == F_RDLCK) + { + lock=1; + if (mi_lock_database(file,F_WRLCK)) + { + if (lock_type == F_RDLCK && my_errno == EDEADLK) + { + printf("%2d: write: deadlock\n",id); fflush(stdout); + return 0; + } + fprintf(stderr,"%2d: Can't lock table (%d)\n",id,my_errno); + return 1; + } + } + bzero((char*) &new_record,sizeof(new_record)); + strmov((char*) new_record.text,"Updated"); + + update=0; + for (i=0 ; i < 100 ; i++) + { + tmp=rnd(100000); + int4store(find,tmp); + if (mi_rkey(file,record.id,1,(uchar*) find, HA_WHOLE_KEY, + HA_READ_KEY_EXACT)) + { + if (my_errno != HA_ERR_KEY_NOT_FOUND) + { + fprintf(stderr,"%2d: Got error %d from read in update\n",id,my_errno); + return 1; + } + else if (mi_rnext(file,record.id,1)) + { + if (my_errno != HA_ERR_END_OF_FILE) + { + fprintf(stderr,"%2d: Got error %d from rnext in update\n", + id,my_errno); + return 1; + } + else if (mi_rprev(file,record.id,1)) + { + if (my_errno != HA_ERR_END_OF_FILE) + { + fprintf(stderr,"%2d: Got error %d from rnext in update\n", + id,my_errno); + return 1; + } + continue; + } + } + } + memcpy(new_record.id, record.id, sizeof(record.id)); + tmp=rnd(20000)+40000; + int4store(new_record.nr,tmp); + if (!mi_update(file,record.id,new_record.id)) + update++; + else + { + if (my_errno != HA_ERR_RECORD_CHANGED && + my_errno != HA_ERR_RECORD_DELETED && + my_errno != HA_ERR_FOUND_DUPP_KEY) + { + fprintf(stderr,"%2d: Got error %d from update\n",id,my_errno); + return 1; + } + } + } + if (lock) + { + if (mi_lock_database(file,F_UNLCK)) + { + fprintf(stderr,"Can't unlock table,id, error%d\n",my_errno); + return 1; + } + } + printf("%2d: update: %5d\n",id,update); fflush(stdout); + return 0; +} + +#include "mi_extrafunc.h" +#else /* _WIN32 */ + +#include <stdio.h> + +int main() +{ + fprintf(stderr,"this test has not been ported to Windows\n"); + return 0; +} + +#endif /* _WIN32 */ diff --git a/storage/myisam/mi_test_all.res b/storage/myisam/mi_test_all.res new file mode 100644 index 00000000..4a22809b --- /dev/null +++ b/storage/myisam/mi_test_all.res @@ -0,0 +1,53 @@ +myisamchk: MyISAM file test1 +myisamchk: warning: Size of indexfile is: 1024 Should be: 2048 +MyISAM-table 'test1' is usable but should be fixed +mi_test2 -s -L -K -R1 -m2000 ; Should give error 135 +Error: 135 in write at record: 1105 +got error: 135 when using MyISAM-database +myisamchk: MyISAM file test2 +myisamchk: warning: Datafile is almost full, 65532 of 65534 used +MyISAM-table 'test2' is usable but should be fixed +Commands Used count Errors Recover errors +open 7 0 0 +write 350 0 0 +update 35 0 0 +delete 350 0 0 +close 7 0 0 +extra 42 0 0 +Total 791 0 0 +Commands Used count Errors Recover errors +open 8 0 0 +write 400 0 0 +update 40 0 0 +delete 400 0 0 +close 8 0 0 +extra 48 0 0 +Total 904 0 0 + +real 0m0.221s +user 0m0.120s +sys 0m0.100s + +real 0m0.222s +user 0m0.140s +sys 0m0.084s + +real 0m0.232s +user 0m0.112s +sys 0m0.120s + +real 0m0.163s +user 0m0.116s +sys 0m0.036s + +real 0m0.159s +user 0m0.136s +sys 0m0.020s + +real 0m0.147s +user 0m0.132s +sys 0m0.016s + +real 0m0.211s +user 0m0.124s +sys 0m0.088s diff --git a/storage/myisam/mi_test_all.sh b/storage/myisam/mi_test_all.sh new file mode 100755 index 00000000..9b385bb9 --- /dev/null +++ b/storage/myisam/mi_test_all.sh @@ -0,0 +1,168 @@ +#!/bin/sh + +# Copyright (C) 2000, 2007 MySQL AB +# Use is subject to license terms +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Library General Public +# License as published by the Free Software Foundation; version 2 +# of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Library General Public License for more details. +# +# You should have received a copy of the GNU Library General Public +# License along with this library; if not, write to the Free +# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1335 USA + +# +# Execute some simple basic tests on the MyISAM library to check if +# things work at all. + +valgrind="valgrind --alignment=8 --leak-check=yes" +silent="-s" +rm -f test1.TMD + +if test -f mi_test1$MACH ; then suffix=$MACH ; else suffix=""; fi +./mi_test1$suffix $silent +./myisamchk$suffix -se test1 +./mi_test1$suffix $silent -N -S +./myisamchk$suffix -se test1 +./mi_test1$suffix $silent -P --checksum +./myisamchk$suffix -se test1 +./mi_test1$suffix $silent -P -N -S +./myisamchk$suffix -se test1 +./mi_test1$suffix $silent -B -N -R2 +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent -a -k 480 --unique +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent -a -N -S -R1 +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent -p -S +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent -p -S -N --unique +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent -p -S -N --key_length=127 --checksum +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent -p -S -N --key_length=128 +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent -p -S --key_length=480 +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent -a -B +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent -a -B --key_length=64 --unique +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent -a -B -k 480 --checksum +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent -a -B -k 480 -N --unique --checksum +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent -a -m +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent -a -m -P --unique --checksum +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent -a -m -P --key_length=480 --key_cache +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent -m -p +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent -w -S --unique +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent -a -w --key_length=64 --checksum +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent -a -w -N --key_length=480 +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent -a -w -S --key_length=480 --checksum +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent -a -b -N +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent -a -b --key_length=480 +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent -p -B --key_length=480 +./myisamchk$suffix -sm test1 + +./mi_test1$suffix $silent --checksum +./myisamchk$suffix -se test1 +./myisamchk$suffix -rs test1 +./myisamchk$suffix -se test1 +./myisamchk$suffix -rqs test1 +./myisamchk$suffix -se test1 +./myisamchk$suffix -rs --correct-checksum test1 +./myisamchk$suffix -se test1 +./myisamchk$suffix -rqs --correct-checksum test1 +./myisamchk$suffix -se test1 +./myisamchk$suffix -ros --correct-checksum test1 +./myisamchk$suffix -se test1 +./myisamchk$suffix -rqos --correct-checksum test1 +./myisamchk$suffix -se test1 + +# check of myisampack / myisamchk +./myisampack$suffix --force -s test1 +# Ignore error for index file +./myisamchk$suffix -es test1 2>&1 > /dev/null +./myisamchk$suffix -rqs test1 +./myisamchk$suffix -es test1 +./myisamchk$suffix -rs test1 +./myisamchk$suffix -es test1 +./myisamchk$suffix -rus test1 +./myisamchk$suffix -es test1 + +./mi_test1$suffix $silent --checksum -S +./myisamchk$suffix -se test1 +./myisamchk$suffix -ros test1 +./myisamchk$suffix -rqs test1 +./myisamchk$suffix -se test1 + +./myisampack$suffix --force -s test1 +./myisamchk$suffix -rqs test1 +./myisamchk$suffix -es test1 +./myisamchk$suffix -rus test1 +./myisamchk$suffix -es test1 + +./mi_test1$suffix $silent --checksum --unique +./myisamchk$suffix -se test1 +./mi_test1$suffix $silent --unique -S +./myisamchk$suffix -se test1 + + +./mi_test1$suffix $silent --key_multiple -N -S +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent --key_multiple -a -p --key_length=480 +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent --key_multiple -a -B --key_length=480 +./myisamchk$suffix -sm test1 +./mi_test1$suffix $silent --key_multiple -P -S +./myisamchk$suffix -sm test1 + +./mi_test2$suffix $silent -L -K -W -P +./myisamchk$suffix -sm test2 +./mi_test2$suffix $silent -L -K -W -P -A +./myisamchk$suffix -sm test2 +./mi_test2$suffix $silent -L -K -W -P -S -R1 -m500 +echo "mi_test2$suffix $silent -L -K -R1 -m2000 ; Should give error 135" +./myisamchk$suffix -sm test2 +./mi_test2$suffix $silent -L -K -R1 -m2000 +./myisamchk$suffix -sm test2 +./mi_test2$suffix $silent -L -K -P -S -R3 -m50 -b1000000 +./myisamchk$suffix -sm test2 +./mi_test2$suffix $silent -L -B +./myisamchk$suffix -sm test2 +./mi_test2$suffix $silent -D -B -c +./myisamchk$suffix -sm test2 +./mi_test2$suffix $silent -m10000 -e8192 -K +./myisamchk$suffix -sm test2 +./mi_test2$suffix $silent -m10000 -e16384 -E16384 -K -L +./myisamchk$suffix -sm test2 + +./mi_test2$suffix $silent -L -K -W -P -m50 -l +./myisamlog$suffix -P +./mi_test2$suffix $silent -L -K -W -P -m50 -l -b100 +./myisamlog$suffix -P +time ./mi_test2$suffix $silent +time ./mi_test2$suffix $silent -K -B +time ./mi_test2$suffix $silent -L -B +time ./mi_test2$suffix $silent -L -K -B +time ./mi_test2$suffix $silent -L -K -W -B +time ./mi_test2$suffix $silent -L -K -W -S -B +time ./mi_test2$suffix $silent -D -K -W -S -B diff --git a/storage/myisam/mi_unique.c b/storage/myisam/mi_unique.c new file mode 100644 index 00000000..ecc2b354 --- /dev/null +++ b/storage/myisam/mi_unique.c @@ -0,0 +1,246 @@ +/* + Copyright (c) 2000, 2010, Oracle and/or its affiliates + Copyright (c) 2020, MariaDB Corporation. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Functions to check if a row is unique */ + +#include "myisamdef.h" +#include <m_ctype.h> + +my_bool mi_check_unique(MI_INFO *info, MI_UNIQUEDEF *def, const uchar *record, + ha_checksum unique_hash, my_off_t disk_pos) +{ + my_off_t lastpos=info->lastpos; + MI_KEYDEF *key= &info->s->keyinfo[def->key]; + uchar *key_buff=info->lastkey2; + DBUG_ENTER("mi_check_unique"); + + /* We need to store the hash value as a key in the record, breaking const */ + mi_unique_store(((uchar*) record)+key->seg->start, unique_hash); + _mi_make_key(info,def->key,key_buff,record,0); + + /* The above changed info->lastkey2. Inform mi_rnext_same(). */ + info->update&= ~HA_STATE_RNEXT_SAME; + + if (_mi_search(info,info->s->keyinfo+def->key,key_buff,MI_UNIQUE_HASH_LENGTH, + SEARCH_FIND,info->s->state.key_root[def->key])) + { + info->page_changed=1; /* Can't optimize read next */ + info->lastpos= lastpos; + DBUG_RETURN(0); /* No matching rows */ + } + + for (;;) + { + if (info->lastpos != disk_pos && + !(*info->s->compare_unique)(info,def,record,info->lastpos)) + { + my_errno=HA_ERR_FOUND_DUPP_UNIQUE; + info->errkey= (int) def->key; + info->dupp_key_pos= info->lastpos; + info->page_changed=1; /* Can't optimize read next */ + info->lastpos=lastpos; + DBUG_PRINT("info",("Found duplicate")); + DBUG_RETURN(1); /* Found identical */ + } + if (_mi_search_next(info,info->s->keyinfo+def->key, info->lastkey, + MI_UNIQUE_HASH_LENGTH, SEARCH_BIGGER, + info->s->state.key_root[def->key]) || + memcmp(info->lastkey, key_buff, MI_UNIQUE_HASH_LENGTH)) + { + info->page_changed=1; /* Can't optimize read next */ + info->lastpos=lastpos; + DBUG_RETURN(0); /* end of tree */ + } + } +} + + +/* + Calculate a hash for a row + + TODO + Add support for bit fields +*/ + +ha_checksum mi_unique_hash(MI_UNIQUEDEF *def, const uchar *record) +{ + const uchar *pos, *end; + ha_checksum crc= 0; + ulong seed1=0, seed2= 4; + HA_KEYSEG *keyseg; + + for (keyseg=def->seg ; keyseg < def->end ; keyseg++) + { + enum ha_base_keytype type=(enum ha_base_keytype) keyseg->type; + uint length=keyseg->length; + + if (keyseg->null_bit) + { + if (record[keyseg->null_pos] & keyseg->null_bit) + { + /* + Change crc in a way different from an empty string or 0. + (This is an optimisation; The code will work even if this isn't + done) + */ + crc=((crc << 8) + 511+ + (crc >> (8*sizeof(ha_checksum)-8))); + continue; + } + } + pos= record+keyseg->start; + if (keyseg->flag & HA_VAR_LENGTH_PART) + { + uint pack_length= keyseg->bit_start; + uint tmp_length= (pack_length == 1 ? (uint) *(uchar*) pos : + uint2korr(pos)); + pos+= pack_length; /* Skip VARCHAR length */ + set_if_smaller(length,tmp_length); + } + else if (keyseg->flag & HA_BLOB_PART) + { + uint tmp_length=_mi_calc_blob_length(keyseg->bit_start,pos); + memcpy((char**) &pos, pos+keyseg->bit_start, sizeof(char*)); + if (!length || length > tmp_length) + length=tmp_length; /* The whole blob */ + } + end= pos+length; + if (type == HA_KEYTYPE_TEXT || type == HA_KEYTYPE_VARTEXT1 || + type == HA_KEYTYPE_VARTEXT2) + { + my_ci_hash_sort(keyseg->charset, + (const uchar*) pos, length, + &seed1, &seed2); + crc^= seed1; + } + else + while (pos != end) + crc=((crc << 8) + + (((uchar) *(uchar*) pos++))) + + (crc >> (8*sizeof(ha_checksum)-8)); + } + return crc; +} + + +/* + compare unique key for two rows + + TODO + Add support for bit fields + + RETURN + 0 if both rows have equal unique value + # Rows are different +*/ + +int mi_unique_comp(MI_UNIQUEDEF *def, const uchar *a, const uchar *b, + my_bool null_are_equal) +{ + const uchar *pos_a, *pos_b, *end; + HA_KEYSEG *keyseg; + + for (keyseg=def->seg ; keyseg < def->end ; keyseg++) + { + enum ha_base_keytype type=(enum ha_base_keytype) keyseg->type; + uint a_length, b_length; + a_length= b_length= keyseg->length; + + /* If part is NULL it's regarded as different */ + if (keyseg->null_bit) + { + uint tmp; + if ((tmp=(a[keyseg->null_pos] & keyseg->null_bit)) != + (uint) (b[keyseg->null_pos] & keyseg->null_bit)) + return 1; + if (tmp) + { + if (!null_are_equal) + return 1; + continue; + } + } + pos_a= a+keyseg->start; + pos_b= b+keyseg->start; + if (keyseg->flag & HA_VAR_LENGTH_PART) + { + uint pack_length= keyseg->bit_start; + if (pack_length == 1) + { + a_length= (uint) *(uchar*) pos_a++; + b_length= (uint) *(uchar*) pos_b++; + } + else + { + a_length= uint2korr(pos_a); + b_length= uint2korr(pos_b); + pos_a+= 2; /* Skip VARCHAR length */ + pos_b+= 2; + } + set_if_smaller(a_length, keyseg->length); /* Safety */ + set_if_smaller(b_length, keyseg->length); /* safety */ + } + else if (keyseg->flag & HA_BLOB_PART) + { + /* Only compare 'length' characters if length != 0 */ + a_length= _mi_calc_blob_length(keyseg->bit_start,pos_a); + b_length= _mi_calc_blob_length(keyseg->bit_start,pos_b); + /* Check that a and b are of equal length */ + if (keyseg->length) + { + /* + This is used in some cases when we are not interested in comparing + the whole length of the blob. + */ + set_if_smaller(a_length, keyseg->length); + set_if_smaller(b_length, keyseg->length); + } + memcpy((char**) &pos_a, pos_a+keyseg->bit_start, sizeof(char*)); + memcpy((char**) &pos_b, pos_b+keyseg->bit_start, sizeof(char*)); + } + if (type == HA_KEYTYPE_TEXT/*The CHAR data type*/) + { + if (ha_compare_char_fixed(keyseg->charset, + (uchar *) pos_a, a_length, + (uchar *) pos_b, b_length, + keyseg->length / keyseg->charset->mbmaxlen, + FALSE/*b_is_prefix*/)) + return 1; + } + else if (type == HA_KEYTYPE_VARTEXT1 || + type == HA_KEYTYPE_VARTEXT2) + { + if (ha_compare_char_varying(keyseg->charset, + (uchar *) pos_a, a_length, + (uchar *) pos_b, b_length, + FALSE/*b_is_prefix*/)) + return 1; + } + else + { + if (a_length != b_length) + return 1; + end= pos_a+a_length; + while (pos_a != end) + { + if (*pos_a++ != *pos_b++) + return 1; + } + } + } + return 0; +} diff --git a/storage/myisam/mi_update.c b/storage/myisam/mi_update.c new file mode 100644 index 00000000..68d76790 --- /dev/null +++ b/storage/myisam/mi_update.c @@ -0,0 +1,241 @@ +/* + Copyright (c) 2000, 2011, Oracle and/or its affiliates + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Update an old row in a MyISAM table */ + +#include "fulltext.h" +#include "rt_index.h" + +int mi_update(register MI_INFO *info, const uchar *oldrec, + const uchar *newrec) +{ + int flag,key_changed,save_errno; + reg3 my_off_t pos; + uint i; + uchar old_key[HA_MAX_KEY_BUFF],*new_key; + my_bool auto_key_changed=0; + ulonglong changed; + MYISAM_SHARE *share=info->s; + ha_checksum UNINIT_VAR(old_checksum); + DBUG_ENTER("mi_update"); + + DBUG_EXECUTE_IF("myisam_pretend_crashed_table_on_usage", + mi_print_error(info->s, HA_ERR_CRASHED); + DBUG_RETURN(my_errno= HA_ERR_CRASHED);); + if (!(info->update & HA_STATE_AKTIV)) + { + DBUG_RETURN(my_errno=HA_ERR_KEY_NOT_FOUND); + } + if (share->options & HA_OPTION_READ_ONLY_DATA) + { + DBUG_RETURN(my_errno=EACCES); + } + if (info->state->key_file_length >= share->base.margin_key_file_length) + { + DBUG_RETURN(my_errno=HA_ERR_INDEX_FILE_FULL); + } + pos=info->lastpos; + if (_mi_readinfo(info,F_WRLCK,1)) + DBUG_RETURN(my_errno); + + if (share->calc_checksum) + old_checksum=info->checksum=(*share->calc_checksum)(info,oldrec); + if ((*share->compare_record)(info,oldrec)) + { + save_errno=my_errno; + goto err_end; /* Record has changed */ + } + + + /* Calculate and check all unique constraints */ + key_changed=0; + for (i=0 ; i < share->state.header.uniques ; i++) + { + MI_UNIQUEDEF *def=share->uniqueinfo+i; + if (mi_unique_comp(def, newrec, oldrec,1) && + mi_check_unique(info, def, newrec, mi_unique_hash(def, newrec), + info->lastpos)) + { + save_errno=my_errno; + goto err_end; + } + } + if (_mi_mark_file_changed(info)) + { + save_errno=my_errno; + goto err_end; + } + + /* Check which keys changed from the original row */ + + new_key=info->lastkey2; + changed=0; + for (i=0 ; i < share->base.keys ; i++) + { + if (mi_is_key_active(share->state.key_map, i)) + { + if (share->keyinfo[i].flag & HA_FULLTEXT ) + { + if (_mi_ft_cmp(info,i,oldrec, newrec)) + { + if ((int) i == info->lastinx) + { + /* + We are changeing the index we are reading on. Mark that + the index data has changed and we need to do a full search + when doing read-next + */ + key_changed|=HA_STATE_WRITTEN; + } + changed|=((ulonglong) 1 << i); + if (_mi_ft_update(info,i, old_key,oldrec,newrec,pos)) + goto err; + } + } + else + { + uint new_length=_mi_make_key(info,i,new_key,newrec,pos); + uint old_length=_mi_make_key(info,i,old_key,oldrec,pos); + + /* The above changed info->lastkey2. Inform mi_rnext_same(). */ + info->update&= ~HA_STATE_RNEXT_SAME; + + if (new_length != old_length || + memcmp((uchar*) old_key,(uchar*) new_key,new_length)) + { + if ((int) i == info->lastinx) + key_changed|=HA_STATE_WRITTEN; /* Mark that keyfile changed */ + changed|=((ulonglong) 1 << i); + share->keyinfo[i].version++; + if (share->keyinfo[i].ck_delete(info,i,old_key,old_length)) goto err; + if (share->keyinfo[i].ck_insert(info,i,new_key,new_length)) goto err; + if (share->base.auto_key == i+1) + auto_key_changed=1; + } + } + } + } + /* + If we are running with external locking, we must update the index file + that something has changed. + */ + if (changed || !my_disable_locking) + key_changed|= HA_STATE_CHANGED; + + if (share->calc_checksum) + { + info->checksum=(*share->calc_checksum)(info,newrec); + /* Store new checksum in index file header */ + key_changed|= HA_STATE_CHANGED; + } + { + /* + Don't update index file if data file is not extended and no status + information changed + */ + MI_STATUS_INFO state; + ha_rows org_split; + my_off_t org_delete_link; + + memcpy((char*) &state, (char*) info->state, sizeof(state)); + org_split= share->state.split; + org_delete_link= share->state.dellink; + if ((*share->update_record)(info,pos,newrec)) + goto err; + if (!key_changed && + (memcmp((char*) &state, (char*) info->state, sizeof(state)) || + org_split != share->state.split || + org_delete_link != share->state.dellink)) + key_changed|= HA_STATE_CHANGED; /* Must update index file */ + } + if (auto_key_changed) + set_if_bigger(info->s->state.auto_increment, + retrieve_auto_increment(info, newrec)); + if (share->calc_checksum) + info->state->checksum+=(info->checksum - old_checksum); + + info->update= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED | HA_STATE_AKTIV | + key_changed); + myisam_log_record(MI_LOG_UPDATE,info,newrec,info->lastpos,0); + /* + Every myisam function that updates myisam table must end with + call to _mi_writeinfo(). If operation (second param of + _mi_writeinfo()) is not 0 it sets share->changed to 1, that is + flags that data has changed. If operation is 0, this function + equals to no-op in this case. + + mi_update() must always pass !0 value as operation, since even if + there is no index change there could be data change. + */ + (void) _mi_writeinfo(info, WRITEINFO_UPDATE_KEYFILE); + if (info->invalidator != 0) + { + DBUG_PRINT("info", ("invalidator... '%s' (update)", info->filename)); + (*info->invalidator)(info->filename); + info->invalidator=0; + } + DBUG_RETURN(0); + +err: + DBUG_PRINT("error",("key: %d errno: %d",i,my_errno)); + save_errno=my_errno; + if (changed) + key_changed|= HA_STATE_CHANGED; + if (my_errno == HA_ERR_FOUND_DUPP_KEY || my_errno == HA_ERR_RECORD_FILE_FULL || + my_errno == HA_ERR_NULL_IN_SPATIAL || my_errno == HA_ERR_OUT_OF_MEM) + { + info->errkey= (int) i; + flag=0; + do + { + if (((ulonglong) 1 << i) & changed) + { + if (share->keyinfo[i].flag & HA_FULLTEXT) + { + if ((flag++ && _mi_ft_del(info,i, new_key,newrec,pos)) || + _mi_ft_add(info,i, old_key,oldrec,pos)) + break; + } + else + { + uint new_length=_mi_make_key(info,i,new_key,newrec,pos); + uint old_length= _mi_make_key(info,i,old_key,oldrec,pos); + if ((flag++ && + share->keyinfo[i].ck_delete(info, i, new_key, new_length)) || + share->keyinfo[i].ck_insert(info, i, old_key, old_length)) + break; + } + } + } while (i-- != 0); + } + else + { + mi_print_error(info->s, HA_ERR_CRASHED); + mi_mark_crashed(info); + } + info->update= (HA_STATE_CHANGED | HA_STATE_AKTIV | HA_STATE_ROW_CHANGED | + key_changed); + + err_end: + myisam_log_record(MI_LOG_UPDATE,info,newrec,info->lastpos,my_errno); + (void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE); + if (save_errno == HA_ERR_KEY_NOT_FOUND) + { + mi_print_error(info->s, HA_ERR_CRASHED); + save_errno=HA_ERR_CRASHED; + } + DBUG_RETURN(my_errno=save_errno); +} /* mi_update */ diff --git a/storage/myisam/mi_write.c b/storage/myisam/mi_write.c new file mode 100644 index 00000000..e8a985a5 --- /dev/null +++ b/storage/myisam/mi_write.c @@ -0,0 +1,1063 @@ +/* Copyright (c) 2000, 2013, Oracle and/or its affiliates + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Write a row to a MyISAM table */ + +#include "fulltext.h" +#include "rt_index.h" + +#define MAX_POINTER_LENGTH 8 + + /* Functions declared in this file */ + +static int w_search(MI_INFO *info,MI_KEYDEF *keyinfo, + uint comp_flag, uchar *key, + uint key_length, my_off_t pos, uchar *father_buff, + uchar *father_keypos, my_off_t father_page, + my_bool insert_last); +static int _mi_balance_page(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *key, + uchar *curr_buff,uchar *father_buff, + uchar *father_keypos,my_off_t father_page); +static uchar *_mi_find_last_pos(MI_KEYDEF *keyinfo, uchar *page, + uchar *key, uint *return_key_length, + uchar **after_key); +int _mi_ck_write_tree(register MI_INFO *info, uint keynr,uchar *key, + uint key_length); +int _mi_ck_write_btree(register MI_INFO *info, uint keynr,uchar *key, + uint key_length); + + /* Write new record to database */ + +int mi_write(MI_INFO *info, const uchar *record) +{ + MYISAM_SHARE *share=info->s; + uint i; + int save_errno; + my_off_t filepos; + uchar *buff; + my_bool lock_tree= share->concurrent_insert; + DBUG_ENTER("mi_write"); + DBUG_PRINT("enter",("isam: %d data: %d",info->s->kfile,info->dfile)); + + DBUG_EXECUTE_IF("myisam_pretend_crashed_table_on_usage", + mi_print_error(info->s, HA_ERR_CRASHED); + DBUG_RETURN(my_errno= HA_ERR_CRASHED);); + + /* it's always a bug to try to write a record with the deleted flag set */ + DBUG_ASSERT(info->s->data_file_type != STATIC_RECORD || *record); + + if (share->options & HA_OPTION_READ_ONLY_DATA) + { + DBUG_RETURN(my_errno=EACCES); + } + if (_mi_readinfo(info,F_WRLCK,1)) + DBUG_RETURN(my_errno); + + filepos= ((share->state.dellink != HA_OFFSET_ERROR && + !info->append_insert_at_end) ? + share->state.dellink : + info->state->data_file_length); + + if (share->base.reloc == (ha_rows) 1 && + share->base.records == (ha_rows) 1 && + info->state->records == (ha_rows) 1) + { /* System file */ + my_errno=HA_ERR_RECORD_FILE_FULL; + goto err2; + } + if (info->state->key_file_length >= share->base.margin_key_file_length) + { + my_errno=HA_ERR_INDEX_FILE_FULL; + goto err2; + } + if (_mi_mark_file_changed(info)) + goto err2; + + /* Calculate and check all unique constraints */ + for (i=0 ; i < share->state.header.uniques ; i++) + { + MI_UNIQUEDEF *def= share->uniqueinfo + i; + if (mi_is_key_active(share->state.key_map, def->key) && + mi_check_unique(info, def, record, mi_unique_hash(def, record), + HA_OFFSET_ERROR)) + goto err2; + } + + /* Write all keys to indextree */ + + buff=info->lastkey2; + for (i=0 ; i < share->base.keys ; i++) + { + if (mi_is_key_active(share->state.key_map, i)) + { + my_bool local_lock_tree= (lock_tree && + !(info->bulk_insert && + is_tree_inited(&info->bulk_insert[i]))); + if (local_lock_tree) + { + mysql_rwlock_wrlock(&share->key_root_lock[i]); + share->keyinfo[i].version++; + } + if (share->keyinfo[i].flag & HA_FULLTEXT ) + { + if (_mi_ft_add(info,i, buff, record, filepos)) + { + if (local_lock_tree) + mysql_rwlock_unlock(&share->key_root_lock[i]); + DBUG_PRINT("error",("Got error: %d on write",my_errno)); + goto err; + } + } + else + { + if (share->keyinfo[i].ck_insert(info,i,buff, + _mi_make_key(info,i,buff,record,filepos))) + { + if (local_lock_tree) + mysql_rwlock_unlock(&share->key_root_lock[i]); + DBUG_PRINT("error",("Got error: %d on write",my_errno)); + goto err; + } + } + + /* The above changed info->lastkey2. Inform mi_rnext_same(). */ + info->update&= ~HA_STATE_RNEXT_SAME; + + if (local_lock_tree) + mysql_rwlock_unlock(&share->key_root_lock[i]); + } + } + if (share->calc_checksum) + info->checksum=(*share->calc_checksum)(info,record); + if (!(info->opt_flag & OPT_NO_ROWS)) + { + if ((*share->write_record)(info,record)) + goto err; + info->state->checksum+=info->checksum; + } + if (share->base.auto_key) + set_if_bigger(info->s->state.auto_increment, + retrieve_auto_increment(info, record)); + info->update= (HA_STATE_CHANGED | HA_STATE_AKTIV | HA_STATE_WRITTEN | + HA_STATE_ROW_CHANGED); + info->state->records++; + myisam_log_record(MI_LOG_WRITE,info,record,filepos,0); + (void) _mi_writeinfo(info, WRITEINFO_UPDATE_KEYFILE); + if (info->invalidator != 0) + { + DBUG_PRINT("info", ("invalidator... '%s' (update)", info->filename)); + (*info->invalidator)(info->filename); + info->invalidator=0; + } + + /* + Update status of the table. We need to do so after each row write + for the log tables, as we want the new row to become visible to + other threads as soon as possible. We don't lock mutex here + (as it is required by pthread memory visibility rules) as (1) it's + not critical to use outdated share->is_log_table value (2) locking + mutex here for every write is too expensive. + */ + if (share->is_log_table) + mi_update_status((void*) info); + + DBUG_RETURN(0); + +err: + save_errno=my_errno; + if (my_errno == HA_ERR_FOUND_DUPP_KEY || my_errno == HA_ERR_RECORD_FILE_FULL || + my_errno == HA_ERR_NULL_IN_SPATIAL || my_errno == HA_ERR_OUT_OF_MEM) + { + if (info->bulk_insert) + { + uint j; + for (j=0 ; j < share->base.keys ; j++) + mi_flush_bulk_insert(info, j); + } + info->errkey= (int) i; + while ( i-- > 0) + { + if (mi_is_key_active(share->state.key_map, i)) + { + my_bool local_lock_tree= (lock_tree && + !(info->bulk_insert && + is_tree_inited(&info->bulk_insert[i]))); + if (local_lock_tree) + mysql_rwlock_wrlock(&share->key_root_lock[i]); + if (share->keyinfo[i].flag & HA_FULLTEXT) + { + if (_mi_ft_del(info,i, buff,record,filepos)) + { + if (local_lock_tree) + mysql_rwlock_unlock(&share->key_root_lock[i]); + break; + } + } + else + { + uint key_length=_mi_make_key(info,i,buff,record,filepos); + if (share->keyinfo[i].ck_delete(info, i, buff, key_length)) + { + if (local_lock_tree) + mysql_rwlock_unlock(&share->key_root_lock[i]); + break; + } + } + if (local_lock_tree) + mysql_rwlock_unlock(&share->key_root_lock[i]); + } + } + } + else + { + mi_print_error(info->s, HA_ERR_CRASHED); + mi_mark_crashed(info); + } + info->update= (HA_STATE_CHANGED | HA_STATE_WRITTEN | HA_STATE_ROW_CHANGED); + my_errno=save_errno; +err2: + save_errno=my_errno; + myisam_log_record(MI_LOG_WRITE,info,record,filepos,my_errno); + (void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE); + DBUG_RETURN(my_errno=save_errno); +} /* mi_write */ + + + /* Write one key to btree */ + +int _mi_ck_write(MI_INFO *info, uint keynr, uchar *key, uint key_length) +{ + DBUG_ENTER("_mi_ck_write"); + + if (info->bulk_insert && is_tree_inited(&info->bulk_insert[keynr])) + { + DBUG_RETURN(_mi_ck_write_tree(info, keynr, key, key_length)); + } + else + { + DBUG_RETURN(_mi_ck_write_btree(info, keynr, key, key_length)); + } +} /* _mi_ck_write */ + + +/********************************************************************** + * Normal insert code * + **********************************************************************/ + +int _mi_ck_write_btree(register MI_INFO *info, uint keynr, uchar *key, + uint key_length) +{ + int error; + uint comp_flag; + MI_KEYDEF *keyinfo=info->s->keyinfo+keynr; + my_off_t *root=&info->s->state.key_root[keynr]; + DBUG_ENTER("_mi_ck_write_btree"); + + if (keyinfo->flag & HA_SORT_ALLOWS_SAME) + comp_flag=SEARCH_BIGGER; /* Put after same key */ + else if (keyinfo->flag & (HA_NOSAME|HA_FULLTEXT)) + { + comp_flag=SEARCH_FIND | SEARCH_UPDATE | SEARCH_INSERT; /* No duplicates */ + if (keyinfo->flag & HA_NULL_ARE_EQUAL) + comp_flag|= SEARCH_NULL_ARE_EQUAL; + } + else + comp_flag=SEARCH_SAME; /* Keys in rec-pos order */ + + error=_mi_ck_real_write_btree(info, keyinfo, key, key_length, + root, comp_flag); + if (info->ft1_to_ft2) + { + if (!error) + error= _mi_ft_convert_to_ft2(info, keynr, key); + delete_dynamic(info->ft1_to_ft2); + my_free(info->ft1_to_ft2); + info->ft1_to_ft2=0; + } + DBUG_RETURN(error); +} /* _mi_ck_write_btree */ + +int _mi_ck_real_write_btree(MI_INFO *info, MI_KEYDEF *keyinfo, + uchar *key, uint key_length, my_off_t *root, uint comp_flag) +{ + int error; + DBUG_ENTER("_mi_ck_real_write_btree"); + /* key_length parameter is used only if comp_flag is SEARCH_FIND */ + if (*root == HA_OFFSET_ERROR || + (error=w_search(info, keyinfo, comp_flag, key, key_length, + *root, (uchar *) 0, (uchar*) 0, + (my_off_t) 0, 1)) > 0) + error=_mi_enlarge_root(info,keyinfo,key,root); + DBUG_RETURN(error); +} /* _mi_ck_real_write_btree */ + + + /* Make a new root with key as only pointer */ + +int _mi_enlarge_root(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *key, + my_off_t *root) +{ + uint t_length,nod_flag; + MI_KEY_PARAM s_temp; + MYISAM_SHARE *share=info->s; + DBUG_ENTER("_mi_enlarge_root"); + + nod_flag= (*root != HA_OFFSET_ERROR) ? share->base.key_reflength : 0; + _mi_kpointer(info,info->buff+2,*root); /* if nod */ + t_length=(*keyinfo->pack_key)(keyinfo,nod_flag,(uchar*) 0, + (uchar*) 0, (uchar*) 0, key,&s_temp); + mi_putint(info->buff,t_length+2+nod_flag,nod_flag); + (*keyinfo->store_key)(keyinfo,info->buff+2+nod_flag,&s_temp); + info->buff_used=info->page_changed=1; /* info->buff is used */ + if ((*root= _mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR || + _mi_write_keypage(info,keyinfo,*root,DFLT_INIT_HITS,info->buff)) + DBUG_RETURN(-1); + DBUG_RETURN(0); +} /* _mi_enlarge_root */ + + + /* + Search after a position for a key and store it there + Returns -1 = error + 0 = ok + 1 = key should be stored in higher tree + */ + +static int w_search(register MI_INFO *info, register MI_KEYDEF *keyinfo, + uint comp_flag, uchar *key, uint key_length, my_off_t page, + uchar *father_buff, uchar *father_keypos, + my_off_t father_page, my_bool insert_last) +{ + int error,flag; + uint nod_flag, search_key_length; + uchar *temp_buff,*keypos; + uchar keybuff[HA_MAX_KEY_BUFF]; + my_bool was_last_key; + my_off_t next_page, dupp_key_pos; + DBUG_ENTER("w_search"); + DBUG_PRINT("enter",("page: %ld", (long) page)); + + search_key_length= (comp_flag & SEARCH_FIND) ? key_length : USE_WHOLE_KEY; + if (!(temp_buff= (uchar*) my_alloca((uint) keyinfo->block_length+ + HA_MAX_KEY_BUFF*2))) + DBUG_RETURN(-1); + if (!_mi_fetch_keypage(info,keyinfo,page,DFLT_INIT_HITS,temp_buff,0)) + goto err; + + flag=(*keyinfo->bin_search)(info,keyinfo,temp_buff,key,search_key_length, + comp_flag, &keypos, keybuff, &was_last_key); + nod_flag=mi_test_if_nod(temp_buff); + if (flag == 0) + { + uint tmp_key_length; + /* get position to record with duplicated key */ + tmp_key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,keybuff); + if (tmp_key_length) + dupp_key_pos=_mi_dpos(info,0,keybuff+tmp_key_length); + else + dupp_key_pos= HA_OFFSET_ERROR; + + if (keyinfo->flag & HA_FULLTEXT) + { + uint off; + int subkeys; + + get_key_full_length_rdonly(off, keybuff); + subkeys=ft_sintXkorr(keybuff+off); + comp_flag=SEARCH_SAME; + if (subkeys >= 0) + { + /* normal word, one-level tree structure */ + flag=(*keyinfo->bin_search)(info, keyinfo, temp_buff, key, + USE_WHOLE_KEY, comp_flag, + &keypos, keybuff, &was_last_key); + } + else + { + /* popular word. two-level tree. going down */ + my_off_t root=dupp_key_pos; + keyinfo=&info->s->ft2_keyinfo; + get_key_full_length_rdonly(off, key); + key+=off; + keypos-=keyinfo->keylength+nod_flag; /* we'll modify key entry 'in vivo' */ + error=_mi_ck_real_write_btree(info, keyinfo, key, 0, + &root, comp_flag); + _mi_dpointer(info, keypos+HA_FT_WLEN, root); + subkeys--; /* should there be underflow protection ? */ + DBUG_ASSERT(subkeys < 0); + ft_intXstore(keypos, subkeys); + if (!error) + error=_mi_write_keypage(info,keyinfo,page,DFLT_INIT_HITS,temp_buff); + my_afree((uchar*) temp_buff); + DBUG_RETURN(error); + } + } + else /* not HA_FULLTEXT, normal HA_NOSAME key */ + { + info->dupp_key_pos= dupp_key_pos; + my_afree((uchar*) temp_buff); + my_errno=HA_ERR_FOUND_DUPP_KEY; + DBUG_RETURN(-1); + } + } + if (flag == MI_FOUND_WRONG_KEY) + DBUG_RETURN(-1); + if (!was_last_key) + insert_last=0; + next_page=_mi_kpos(nod_flag,keypos); + if (next_page == HA_OFFSET_ERROR || + (error=w_search(info, keyinfo, comp_flag, key, key_length, next_page, + temp_buff, keypos, page, insert_last)) >0) + { + error=_mi_insert(info,keyinfo,key,temp_buff,keypos,keybuff,father_buff, + father_keypos,father_page, insert_last); + if (_mi_write_keypage(info,keyinfo,page,DFLT_INIT_HITS,temp_buff)) + goto err; + } + my_afree((uchar*) temp_buff); + DBUG_RETURN(error); +err: + my_afree((uchar*) temp_buff); + DBUG_PRINT("exit",("Error: %d",my_errno)); + DBUG_RETURN (-1); +} /* w_search */ + + +/* + Insert new key. + + SYNOPSIS + _mi_insert() + info Open table information. + keyinfo Key definition information. + key New key. + anc_buff Key page (beginning). + key_pos Position in key page where to insert. + key_buff Copy of previous key. + father_buff parent key page for balancing. + father_key_pos position in parent key page for balancing. + father_page position of parent key page in file. + insert_last If to append at end of page. + + DESCRIPTION + Insert new key at right of key_pos. + + RETURN + 2 if key contains key to upper level. + 0 OK. + < 0 Error. +*/ + +int _mi_insert(register MI_INFO *info, register MI_KEYDEF *keyinfo, + uchar *key, uchar *anc_buff, uchar *key_pos, uchar *key_buff, + uchar *father_buff, uchar *father_key_pos, my_off_t father_page, + my_bool insert_last) +{ + uint a_length,nod_flag; + int t_length; + uchar *endpos, *prev_key; + MI_KEY_PARAM s_temp; + DBUG_ENTER("_mi_insert"); + DBUG_PRINT("enter",("key_pos: %p", key_pos)); + DBUG_EXECUTE("key",_mi_print_key(DBUG_FILE,keyinfo->seg,key,USE_WHOLE_KEY);); + + nod_flag=mi_test_if_nod(anc_buff); + a_length=mi_getint(anc_buff); + endpos= anc_buff+ a_length; + prev_key=(key_pos == anc_buff+2+nod_flag ? (uchar*) 0 : key_buff); + t_length=(*keyinfo->pack_key)(keyinfo,nod_flag, + (key_pos == endpos ? (uchar*) 0 : key_pos), + prev_key, prev_key, + key,&s_temp); +#ifndef DBUG_OFF + if (key_pos != anc_buff+2+nod_flag && (keyinfo->flag & + (HA_BINARY_PACK_KEY | HA_PACK_KEY))) + { + DBUG_DUMP("prev_key",(uchar*) key_buff,_mi_keylength(keyinfo,key_buff)); + } + if (keyinfo->flag & HA_PACK_KEY) + { + DBUG_PRINT("test",("t_length: %d ref_len: %d", + t_length,s_temp.ref_length)); + DBUG_PRINT("test",("n_ref_len: %d n_length: %d key_pos: %p", + s_temp.n_ref_length,s_temp.n_length, s_temp.key)); + } +#endif + if (t_length > 0) + { + if (t_length >= keyinfo->maxlength*2+MAX_POINTER_LENGTH) + { + mi_print_error(info->s, HA_ERR_CRASHED); + my_errno=HA_ERR_CRASHED; + DBUG_RETURN(-1); + } + bmove_upp((uchar*) endpos+t_length,(uchar*) endpos,(uint) (endpos-key_pos)); + } + else + { + if (-t_length >= keyinfo->maxlength*2+MAX_POINTER_LENGTH) + { + mi_print_error(info->s, HA_ERR_CRASHED); + my_errno=HA_ERR_CRASHED; + DBUG_RETURN(-1); + } + bmove(key_pos,key_pos-t_length,(uint) (endpos-key_pos)+t_length); + } + (*keyinfo->store_key)(keyinfo,key_pos,&s_temp); + a_length+=t_length; + mi_putint(anc_buff,a_length,nod_flag); + if (a_length <= keyinfo->block_length) + { + if (keyinfo->block_length - a_length < 32 && + keyinfo->flag & HA_FULLTEXT && key_pos == endpos && + info->s->base.key_reflength <= info->s->rec_reflength && + info->s->options & (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) + { + /* + Normal word. One-level tree. Page is almost full. + Let's consider converting. + We'll compare 'key' and the first key at anc_buff + */ + uchar *a=key, *b=anc_buff+2+nod_flag; + uint alen, blen, ft2len=info->s->ft2_keyinfo.keylength; + /* the very first key on the page is always unpacked */ + DBUG_ASSERT((*b & 128) == 0); +#if HA_FT_MAXLEN >= 127 + blen= mi_uint2korr(b); b+=2; +#else + blen= *b++; +#endif + get_key_length(alen,a); + DBUG_ASSERT(info->ft1_to_ft2==0); + if (alen == blen && + ha_compare_word(keyinfo->seg->charset, + a, alen, + b, blen) == 0) + { + /* yup. converting */ + info->ft1_to_ft2=(DYNAMIC_ARRAY *) + my_malloc(mi_key_memory_MI_INFO_ft1_to_ft2, + sizeof(DYNAMIC_ARRAY), MYF(MY_WME)); + my_init_dynamic_array(mi_key_memory_MI_INFO_ft1_to_ft2, + info->ft1_to_ft2, ft2len, 300, 50, MYF(0)); + + /* + now, adding all keys from the page to dynarray + if the page is a leaf (if not keys will be deleted later) + */ + if (!nod_flag) + { + /* let's leave the first key on the page, though, because + we cannot easily dispatch an empty page here */ + b+=blen+ft2len+2; + for (a=anc_buff+a_length ; b < a ; b+=ft2len+2) + { + if (insert_dynamic(info->ft1_to_ft2, b)) + { + mi_print_error(info->s, HA_ERR_OUT_OF_MEM); + my_errno= HA_ERR_OUT_OF_MEM; + DBUG_RETURN(-1); + } + } + + /* fixing the page's length - it contains only one key now */ + mi_putint(anc_buff,2+blen+ft2len+2,0); + } + /* the rest will be done when we're back from recursion */ + } + } + DBUG_RETURN(0); /* There is room on page */ + } + /* Page is full */ + if (nod_flag) + insert_last=0; + if (!(keyinfo->flag & (HA_VAR_LENGTH_KEY | HA_BINARY_PACK_KEY)) && + father_buff && !insert_last) + DBUG_RETURN(_mi_balance_page(info,keyinfo,key,anc_buff,father_buff, + father_key_pos,father_page)); + DBUG_RETURN(_mi_split_page(info,keyinfo,key,anc_buff,key_buff, insert_last)); +} /* _mi_insert */ + + + /* split a full page in two and assign emerging item to key */ + +int _mi_split_page(register MI_INFO *info, register MI_KEYDEF *keyinfo, + uchar *key, uchar *buff, uchar *key_buff, + my_bool insert_last_key) +{ + uint length,a_length,key_ref_length,t_length,nod_flag,key_length; + uchar *key_pos,*pos, *UNINIT_VAR(after_key); + my_off_t new_pos; + MI_KEY_PARAM s_temp; + DBUG_ENTER("mi_split_page"); + DBUG_DUMP("buff",(uchar*) buff,mi_getint(buff)); + + if (info->s->keyinfo+info->lastinx == keyinfo) + info->page_changed=1; /* Info->buff is used */ + info->buff_used=1; + nod_flag=mi_test_if_nod(buff); + key_ref_length=2+nod_flag; + if (insert_last_key) + key_pos=_mi_find_last_pos(keyinfo,buff,key_buff, &key_length, &after_key); + else + key_pos=_mi_find_half_pos(nod_flag,keyinfo,buff,key_buff, &key_length, + &after_key); + if (!key_pos) + DBUG_RETURN(-1); + + length=(uint) (key_pos-buff); + a_length=mi_getint(buff); + mi_putint(buff,length,nod_flag); + + key_pos=after_key; + if (nod_flag) + { + DBUG_PRINT("test",("Splitting nod")); + pos=key_pos-nod_flag; + memcpy((uchar*) info->buff+2,(uchar*) pos,(size_t) nod_flag); + } + + /* Move middle item to key and pointer to new page */ + if ((new_pos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR) + DBUG_RETURN(-1); + _mi_kpointer(info,_mi_move_key(keyinfo,key,key_buff),new_pos); + + /* Store new page */ + if (!(*keyinfo->get_key)(keyinfo,nod_flag,&key_pos,key_buff)) + DBUG_RETURN(-1); + + t_length=(*keyinfo->pack_key)(keyinfo,nod_flag,(uchar *) 0, + (uchar*) 0, (uchar*) 0, + key_buff, &s_temp); + length=(uint) ((buff+a_length)-key_pos); + memcpy((uchar*) info->buff+key_ref_length+t_length,(uchar*) key_pos, + (size_t) length); + (*keyinfo->store_key)(keyinfo,info->buff+key_ref_length,&s_temp); + mi_putint(info->buff,length+t_length+key_ref_length,nod_flag); + + if (_mi_write_keypage(info,keyinfo,new_pos,DFLT_INIT_HITS,info->buff)) + DBUG_RETURN(-1); + DBUG_DUMP("key",(uchar*) key,_mi_keylength(keyinfo,key)); + DBUG_RETURN(2); /* Middle key up */ +} /* _mi_split_page */ + + + /* + Calculate how to much to move to split a page in two + Returns pointer to start of key. + key will contain the key. + return_key_length will contain the length of key + after_key will contain the position to where the next key starts + */ + +uchar *_mi_find_half_pos(uint nod_flag, MI_KEYDEF *keyinfo, uchar *page, + uchar *key, uint *return_key_length, + uchar **after_key) +{ + uint keys,length,key_ref_length; + uchar *end,*lastpos; + DBUG_ENTER("_mi_find_half_pos"); + + key_ref_length=2+nod_flag; + length=mi_getint(page)-key_ref_length; + page+=key_ref_length; + if (!(keyinfo->flag & + (HA_PACK_KEY | HA_SPACE_PACK_USED | HA_VAR_LENGTH_KEY | + HA_BINARY_PACK_KEY))) + { + key_ref_length=keyinfo->keylength+nod_flag; + keys=length/(key_ref_length*2); + *return_key_length=keyinfo->keylength; + end=page+keys*key_ref_length; + *after_key=end+key_ref_length; + memcpy(key,end,key_ref_length); + DBUG_RETURN(end); + } + + end=page+length/2-key_ref_length; /* This is aprox. half */ + *key='\0'; + do + { + lastpos=page; + if (!(length=(*keyinfo->get_key)(keyinfo,nod_flag,&page,key))) + DBUG_RETURN(0); + } while (page < end); + *return_key_length=length; + *after_key=page; + DBUG_PRINT("exit",("returns: %p page: %p half: %p", + lastpos, page, end)); + DBUG_RETURN(lastpos); +} /* _mi_find_half_pos */ + + +/* + Split buffer at last key + Returns pointer to the start of the key before the last key + key will contain the last key +*/ + +static uchar *_mi_find_last_pos(MI_KEYDEF *keyinfo, uchar *page, + uchar *key, uint *return_key_length, + uchar **after_key) +{ + uint keys, length, UNINIT_VAR(last_length), key_ref_length; + uchar *end,*lastpos,*prevpos; + uchar key_buff[HA_MAX_KEY_BUFF]; + DBUG_ENTER("_mi_find_last_pos"); + + key_ref_length=2; + length=mi_getint(page)-key_ref_length; + page+=key_ref_length; + if (!(keyinfo->flag & + (HA_PACK_KEY | HA_SPACE_PACK_USED | HA_VAR_LENGTH_KEY | + HA_BINARY_PACK_KEY))) + { + keys=length/keyinfo->keylength-2; + *return_key_length=length=keyinfo->keylength; + end=page+keys*length; + *after_key=end+length; + memcpy(key,end,length); + DBUG_RETURN(end); + } + + end=page+length-key_ref_length; + DBUG_ASSERT(page < end); + *key='\0'; + length=0; + lastpos=page; + + do + { + prevpos=lastpos; lastpos=page; + last_length=length; + memcpy(key, key_buff, length); /* previous key */ + if (!(length=(*keyinfo->get_key)(keyinfo,0,&page,key_buff))) + { + mi_print_error(keyinfo->share, HA_ERR_CRASHED); + my_errno=HA_ERR_CRASHED; + DBUG_RETURN(0); + } + } while (page < end); + + *return_key_length=last_length; + *after_key=lastpos; + DBUG_PRINT("exit",("returns: %p page: %p end: %p", + prevpos, page, end)); + DBUG_RETURN(prevpos); +} /* _mi_find_last_pos */ + + + /* Balance page with not packed keys with page on right/left */ + /* returns 0 if balance was done */ + +static int _mi_balance_page(register MI_INFO *info, MI_KEYDEF *keyinfo, + uchar *key, uchar *curr_buff, uchar *father_buff, + uchar *father_key_pos, my_off_t father_page) +{ + my_bool right; + uint k_length,father_length,father_keylength,nod_flag,curr_keylength, + right_length,left_length,new_right_length,new_left_length,extra_length, + length,keys; + uchar *pos,*buff,*extra_buff; + my_off_t next_page,new_pos; + uchar tmp_part_key[HA_MAX_KEY_BUFF]; + DBUG_ENTER("_mi_balance_page"); + + k_length=keyinfo->keylength; + father_length=mi_getint(father_buff); + father_keylength=k_length+info->s->base.key_reflength; + nod_flag=mi_test_if_nod(curr_buff); + curr_keylength=k_length+nod_flag; + info->page_changed=1; + + if ((father_key_pos != father_buff+father_length && + (info->state->records & 1)) || + father_key_pos == father_buff+2+info->s->base.key_reflength) + { + right=1; + next_page= _mi_kpos(info->s->base.key_reflength, + father_key_pos+father_keylength); + buff=info->buff; + DBUG_PRINT("test",("use right page: %lu", (ulong) next_page)); + } + else + { + right=0; + father_key_pos-=father_keylength; + next_page= _mi_kpos(info->s->base.key_reflength,father_key_pos); + /* Fix that curr_buff is to left */ + buff=curr_buff; curr_buff=info->buff; + DBUG_PRINT("test",("use left page: %lu", (ulong) next_page)); + } /* father_key_pos ptr to parting key */ + + if (!_mi_fetch_keypage(info,keyinfo,next_page,DFLT_INIT_HITS,info->buff,0)) + goto err; + DBUG_DUMP("next",(uchar*) info->buff,mi_getint(info->buff)); + + /* Test if there is room to share keys */ + + left_length=mi_getint(curr_buff); + right_length=mi_getint(buff); + keys=(left_length+right_length-4-nod_flag*2)/curr_keylength; + + if ((right ? right_length : left_length) + curr_keylength <= + keyinfo->block_length) + { /* Merge buffs */ + new_left_length=2+nod_flag+(keys/2)*curr_keylength; + new_right_length=2+nod_flag+((keys+1)/2)*curr_keylength; + mi_putint(curr_buff,new_left_length,nod_flag); + mi_putint(buff,new_right_length,nod_flag); + + if (left_length < new_left_length) + { /* Move keys buff -> leaf */ + pos=curr_buff+left_length; + memcpy((uchar*) pos,(uchar*) father_key_pos, (size_t) k_length); + memcpy((uchar*) pos+k_length, (uchar*) buff+2, + (size_t) (length=new_left_length - left_length - k_length)); + pos=buff+2+length; + memcpy((uchar*) father_key_pos,(uchar*) pos,(size_t) k_length); + bmove((uchar*) buff + 2, (uchar*) pos + k_length, new_right_length - 2); + } + else + { /* Move keys -> buff */ + + bmove_upp((uchar*) buff+new_right_length,(uchar*) buff+right_length, + right_length-2); + length=new_right_length-right_length-k_length; + memcpy((uchar*) buff+2+length,father_key_pos,(size_t) k_length); + pos=curr_buff+new_left_length; + memcpy((uchar*) father_key_pos,(uchar*) pos,(size_t) k_length); + memcpy((uchar*) buff+2,(uchar*) pos+k_length,(size_t) length); + } + + if (_mi_write_keypage(info,keyinfo,next_page,DFLT_INIT_HITS,info->buff) || + _mi_write_keypage(info,keyinfo,father_page,DFLT_INIT_HITS,father_buff)) + goto err; + DBUG_RETURN(0); + } + + /* curr_buff[] and buff[] are full, lets split and make new nod */ + + extra_buff=info->buff+info->s->base.max_key_block_length; + new_left_length=new_right_length=2+nod_flag+(keys+1)/3*curr_keylength; + if (keys == 5) /* Too few keys to balance */ + new_left_length-=curr_keylength; + extra_length=nod_flag+left_length+right_length- + new_left_length-new_right_length-curr_keylength; + DBUG_PRINT("info",("left_length: %d right_length: %d new_left_length: %d new_right_length: %d extra_length: %d", + left_length, right_length, + new_left_length, new_right_length, + extra_length)); + mi_putint(curr_buff,new_left_length,nod_flag); + mi_putint(buff,new_right_length,nod_flag); + mi_putint(extra_buff,extra_length+2,nod_flag); + + /* move first largest keys to new page */ + pos=buff+right_length-extra_length; + memcpy((uchar*) extra_buff+2,pos,(size_t) extra_length); + /* Save new parting key */ + memcpy(tmp_part_key, pos-k_length,k_length); + /* Make place for new keys */ + bmove_upp((uchar*) buff+new_right_length,(uchar*) pos-k_length, + right_length-extra_length-k_length-2); + /* Copy keys from left page */ + pos= curr_buff+new_left_length; + memcpy((uchar*) buff+2,(uchar*) pos+k_length, + (size_t) (length=left_length-new_left_length-k_length)); + /* Copy old parting key */ + memcpy((uchar*) buff+2+length,father_key_pos,(size_t) k_length); + + /* Move new parting keys up to caller */ + memcpy((uchar*) (right ? key : father_key_pos),pos,(size_t) k_length); + memcpy((uchar*) (right ? father_key_pos : key),tmp_part_key, k_length); + + if ((new_pos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR) + goto err; + _mi_kpointer(info,key+k_length,new_pos); + if (_mi_write_keypage(info,keyinfo,(right ? new_pos : next_page), + DFLT_INIT_HITS,info->buff) || + _mi_write_keypage(info,keyinfo,(right ? next_page : new_pos), + DFLT_INIT_HITS,extra_buff)) + goto err; + + DBUG_RETURN(1); /* Middle key up */ + +err: + DBUG_RETURN(-1); +} /* _mi_balance_page */ + +/********************************************************************** + * Bulk insert code * + **********************************************************************/ + +typedef struct { + MI_INFO *info; + uint keynr; +} bulk_insert_param; + +int _mi_ck_write_tree(register MI_INFO *info, uint keynr, uchar *key, + uint key_length) +{ + int error; + DBUG_ENTER("_mi_ck_write_tree"); + + error= tree_insert(&info->bulk_insert[keynr], key, + key_length + info->s->rec_reflength, + info->bulk_insert[keynr].custom_arg) ? 0 : HA_ERR_OUT_OF_MEM ; + + DBUG_RETURN(error); +} /* _mi_ck_write_tree */ + + +/* typeof(_mi_keys_compare)=qsort_cmp2 */ + +static int keys_compare(bulk_insert_param *param, uchar *key1, uchar *key2) +{ + uint not_used[2]; + return ha_key_cmp(param->info->s->keyinfo[param->keynr].seg, + key1, key2, USE_WHOLE_KEY, SEARCH_SAME, + not_used); +} + + +static int keys_free(void* key_arg, TREE_FREE mode, void *param_arg) +{ + /* + Probably I can use info->lastkey here, but I'm not sure, + and to be safe I'd better use local lastkey. + */ + bulk_insert_param *param= (bulk_insert_param*)param_arg; + uchar lastkey[HA_MAX_KEY_BUFF], *key= (uchar*)key_arg; + uint keylen; + MI_KEYDEF *keyinfo; + + switch (mode) { + case free_init: + if (param->info->s->concurrent_insert) + { + mysql_rwlock_wrlock(¶m->info->s->key_root_lock[param->keynr]); + param->info->s->keyinfo[param->keynr].version++; + } + return 0; + case free_free: + keyinfo=param->info->s->keyinfo+param->keynr; + keylen=_mi_keylength(keyinfo, key); + memcpy(lastkey, key, keylen); + _mi_ck_write_btree(param->info, param->keynr, lastkey, + keylen - param->info->s->rec_reflength); + return 0; + case free_end: + if (param->info->s->concurrent_insert) + mysql_rwlock_unlock(¶m->info->s->key_root_lock[param->keynr]); + return 0; + } + return 0; +} + + +int mi_init_bulk_insert(MI_INFO *info, size_t cache_size, ha_rows rows) +{ + MYISAM_SHARE *share=info->s; + MI_KEYDEF *key=share->keyinfo; + bulk_insert_param *params; + uint i, num_keys, total_keylength; + ulonglong key_map; + DBUG_ENTER("_mi_init_bulk_insert"); + DBUG_PRINT("enter",("cache_size: %lu", (ulong) cache_size)); + + DBUG_ASSERT(!info->bulk_insert && + (!rows || rows >= MI_MIN_ROWS_TO_USE_BULK_INSERT)); + + mi_clear_all_keys_active(key_map); + for (i=total_keylength=num_keys=0 ; i < share->base.keys ; i++) + { + if (! (key[i].flag & HA_NOSAME) && (share->base.auto_key != i + 1) && + mi_is_key_active(share->state.key_map, i)) + { + num_keys++; + mi_set_key_active(key_map, i); + total_keylength+=key[i].maxlength+TREE_ELEMENT_EXTRA_SIZE; + } + } + + if (num_keys==0 || + num_keys * (size_t) MI_MIN_SIZE_BULK_INSERT_TREE > cache_size) + DBUG_RETURN(0); + + if (rows && rows*total_keylength < cache_size) + cache_size= (size_t) rows; + else + cache_size/=total_keylength*16; + + info->bulk_insert=(TREE *) + my_malloc(mi_key_memory_MI_INFO_bulk_insert, + (sizeof(TREE)*share->base.keys+ + sizeof(bulk_insert_param)*num_keys),MYF(0)); + + if (!info->bulk_insert) + DBUG_RETURN(HA_ERR_OUT_OF_MEM); + + params=(bulk_insert_param *)(info->bulk_insert+share->base.keys); + for (i=0 ; i < share->base.keys ; i++) + { + if (mi_is_key_active(key_map, i)) + { + params->info=info; + params->keynr=i; + /* Only allocate a 16'th of the buffer at a time */ + init_tree(&info->bulk_insert[i], + cache_size * key[i].maxlength, + cache_size * key[i].maxlength, 0, + (qsort_cmp2)keys_compare, keys_free, (void *)params++, MYF(0)); + } + else + info->bulk_insert[i].root=0; + } + + DBUG_RETURN(0); +} + +void mi_flush_bulk_insert(MI_INFO *info, uint inx) +{ + if (info->bulk_insert) + { + if (is_tree_inited(&info->bulk_insert[inx])) + reset_tree(&info->bulk_insert[inx]); + } +} + +int mi_end_bulk_insert(MI_INFO *info, my_bool abort) +{ + int first_error= 0; + if (info->bulk_insert) + { + uint i; + for (i=0 ; i < info->s->base.keys ; i++) + { + if (is_tree_inited(& info->bulk_insert[i])) + { + int error; + if ((error= delete_tree(& info->bulk_insert[i], abort))) + { + first_error= first_error ? first_error : error; + abort= 1; + } + } + } + my_free(info->bulk_insert); + info->bulk_insert=0; + } + return first_error; +} diff --git a/storage/myisam/myisam_ftdump.c b/storage/myisam/myisam_ftdump.c new file mode 100644 index 00000000..f2dc19c5 --- /dev/null +++ b/storage/myisam/myisam_ftdump.c @@ -0,0 +1,275 @@ +/* + Copyright (c) 2001, 2010, Oracle and/or its affiliates + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Written by Sergei A. Golubchik, who has a shared copyright to this code + added support for long options (my_getopt) 22.5.2002 by Jani Tolonen */ + +#include "ftdefs.h" +#include <my_getopt.h> + +static void usage(); +static void complain(int val); +static my_bool get_one_option(const struct my_option *, const char *, + const char *); + +static int count=0, stats=0, dump=0, lstats=0; +static my_bool verbose; +static char *query=NULL; +static uint lengths[256]; + +#define MAX_LEN (HA_FT_MAXBYTELEN+10) +#define HOW_OFTEN_TO_WRITE 10000 + +static struct my_option my_long_options[] = +{ + {"help", 'h', "Display help and exit.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"help", '?', "Synonym for -h.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"count", 'c', "Calculate per-word stats (counts and global weights).", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"dump", 'd', "Dump index (incl. data offsets and word weights).", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"length", 'l', "Report length distribution.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"stats", 's', "Report global stats.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"verbose", 'v', "Be verbose.", + &verbose, &verbose, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + + +int main(int argc,char *argv[]) +{ + int error=0; + uint keylen, keylen2=0, inx, doc_cnt=0; + float weight= 1.0; + double gws, min_gws=0, avg_gws=0; + MI_INFO *info; + char buf[MAX_LEN], buf2[MAX_LEN], buf_maxlen[MAX_LEN], buf_min_gws[MAX_LEN]; + ulong total=0, maxlen=0, uniq=0, max_doc_cnt=0; + struct { MI_INFO *info; } aio0, *aio=&aio0; /* for GWS_IN_USE */ + + MY_INIT(argv[0]); + if ((error= handle_options(&argc, &argv, my_long_options, get_one_option))) + exit(error); + if (count || dump) + verbose=0; + if (!count && !dump && !lstats && !query) + stats=1; + + if (verbose) + setbuf(stdout,NULL); + + if (argc < 2) + usage(); + + { + char *end; + inx= (uint) strtoll(argv[1], &end, 10); + if (*end) + usage(); + } + + init_key_cache(dflt_key_cache, MI_KEY_BLOCK_LENGTH, KEY_BUFFER_INIT, 0, 0, 0, 0); + + if (!(info=mi_open(argv[0], O_RDONLY, + HA_OPEN_ABORT_IF_LOCKED|HA_OPEN_FROM_SQL_LAYER))) + { + error=my_errno; + goto err; + } + + *buf2=0; + aio->info=info; + + if ((inx >= info->s->base.keys) || + !(info->s->keyinfo[inx].flag & HA_FULLTEXT)) + { + printf("Key %d in table %s is not a FULLTEXT key\n", inx, info->filename); + goto err; + } + + mi_lock_database(info, F_EXTRA_LCK); + + info->lastpos= HA_OFFSET_ERROR; + info->update|= HA_STATE_PREV_FOUND; + + while (!(error=mi_rnext(info,NULL,inx))) + { + FT_WEIGTH subkeys; + keylen=*(info->lastkey); + + subkeys.i =ft_sintXkorr(info->lastkey+keylen+1); + if (subkeys.i >= 0) + weight= subkeys.f; + + snprintf(buf,MAX_LEN,"%.*s",(int) keylen,info->lastkey+1); + my_casedn_str(default_charset_info,buf); + total++; + lengths[keylen]++; + + if (count || stats) + { + if (strcmp(buf, buf2)) + { + if (*buf2) + { + uniq++; + avg_gws+=gws=GWS_IN_USE; + if (count) + printf("%9u %20.7f %s\n",doc_cnt,gws,buf2); + if (maxlen<keylen2) + { + maxlen=keylen2; + strmov(buf_maxlen, buf2); + } + if (max_doc_cnt < doc_cnt) + { + max_doc_cnt=doc_cnt; + strmov(buf_min_gws, buf2); + min_gws=gws; + } + } + strmov(buf2, buf); + keylen2=keylen; + doc_cnt=0; + } + doc_cnt+= (subkeys.i >= 0 ? 1 : -subkeys.i); + } + if (dump) + { + if (subkeys.i >= 0) + printf("%9lx %20.7f %s\n", (long) info->lastpos,weight,buf); + else + printf("%9lx => %17d %s\n",(long) info->lastpos,-subkeys.i,buf); + } + if (verbose && (total%HOW_OFTEN_TO_WRITE)==0) + printf("%10ld\r",total); + } + mi_lock_database(info, F_UNLCK); + + if (count || stats) + { + if (*buf2) + { + uniq++; + avg_gws+=gws=GWS_IN_USE; + if (count) + printf("%9u %20.7f %s\n",doc_cnt,gws,buf2); + if (maxlen<keylen2) + { + maxlen=keylen2; + strmov(buf_maxlen, buf2); + } + if (max_doc_cnt < doc_cnt) + { + max_doc_cnt=doc_cnt; + strmov(buf_min_gws, buf2); + min_gws=gws; + } + } + } + + if (stats) + { + count=0; + for (inx=0;inx<256;inx++) + { + count+=lengths[inx]; + if ((ulong) count >= total/2) + break; + } + printf("Total rows: %lu\nTotal words: %lu\n" + "Unique words: %lu\nLongest word: %lu chars (%s)\n" + "Median length: %u\n" + "Average global weight: %f\n" + "Most common word: %lu times, weight: %f (%s)\n", + (long) info->state->records, total, uniq, maxlen, buf_maxlen, + inx, avg_gws/uniq, max_doc_cnt, min_gws, buf_min_gws); + } + if (lstats) + { + count=0; + for (inx=0; inx<256; inx++) + { + count+=lengths[inx]; + if (count && lengths[inx]) + printf("%3u: %10lu %5.2f%% %20lu %4.1f%%\n", inx, + (ulong) lengths[inx],100.0*lengths[inx]/total,(ulong) count, + 100.0*count/total); + } + } + +err: + if (error && error != HA_ERR_END_OF_FILE) + printf("got error %d\n",my_errno); + if (info) + mi_close(info); + return 0; +} + + +static my_bool +get_one_option(const struct my_option *opt, + const char *argument __attribute__((unused)), + const char *filename __attribute__((unused))) +{ + switch(opt->id) { + case 'd': + dump=1; + complain(count || query); + break; + case 's': + stats=1; + complain(query!=0); + break; + case 'c': + count= 1; + complain(dump || query); + break; + case 'l': + lstats=1; + complain(query!=0); + break; + case '?': + case 'h': + usage(); + } + return 0; +} + + +static void usage() +{ + printf("Use: myisam_ftdump <table_name> <index_num>\n"); + my_print_help(my_long_options); + my_print_variables(my_long_options); + exit(1); +} + + +static void complain(int val) /* Kinda assert :-) */ +{ + if (val) + { + printf("You cannot use these options together!\n"); + exit(1); + } +} + +#include "mi_extrafunc.h" diff --git a/storage/myisam/myisamchk.c b/storage/myisam/myisamchk.c new file mode 100644 index 00000000..a5777527 --- /dev/null +++ b/storage/myisam/myisamchk.c @@ -0,0 +1,1819 @@ +/* Copyright (c) 2000, 2012, Oracle and/or its affiliates. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Describe, check and repair of MyISAM tables */ + +#include "fulltext.h" +#include "my_default.h" +#include <m_ctype.h> +#include <stdarg.h> +#include <my_getopt.h> +#include <my_bit.h> + +static uint decode_bits; +static char **default_argv; +static const char *load_default_groups[]= { "myisamchk", 0 }; +static char *set_collation_name, *opt_tmpdir; +static CHARSET_INFO *set_collation; +static long opt_myisam_block_size; +static long opt_key_cache_block_size; +static int stopwords_inited= 0; +static MY_TMPDIR myisamchk_tmpdir; + +static const char *type_names[]= +{ "impossible","char","binary", "short", "long", "float", + "double","number","unsigned short", + "unsigned long","longlong","ulonglong","int24", + "uint24","int8","varchar", "varbin","?", + "?"}; + +static const char *prefix_packed_txt="packed ", + *bin_packed_txt="prefix ", + *diff_txt="stripped ", + *null_txt="NULL", + *blob_txt="BLOB "; + +static const char *field_pack[]= +{"","no endspace", "no prespace", + "no zeros", "blob", "constant", "table-lockup", + "always zero","varchar","unique-hash","?","?"}; + +static const char *myisam_stats_method_str="nulls_unequal"; + +static void get_options(int *argc,char * * *argv); +static void print_version(void); +static void usage(void); +static int myisamchk(HA_CHECK *param, char *filename); +static void descript(HA_CHECK *param, register MI_INFO *info, char * name); +static int mi_sort_records(HA_CHECK *param, register MI_INFO *info, + char * name, uint sort_key, + my_bool write_info, my_bool update_index); +static int sort_record_index(MI_SORT_PARAM *sort_param, MI_INFO *info, + MI_KEYDEF *keyinfo, + my_off_t page,uchar *buff,uint sortkey, + File new_file, my_bool update_index); + +HA_CHECK check_param; + + /* Main program */ + +int main(int argc, char **argv) +{ + int error; + uchar rc; + MY_INIT(argv[0]); + my_progname_short= "myisamchk"; + + myisamchk_init(&check_param); + check_param.opt_lock_memory=1; /* Lock memory if possible */ + check_param.using_global_keycache = 0; + get_options(&argc,(char***) &argv); + myisam_quick_table_bits=decode_bits; + error=0; + while (--argc >= 0) + { + int new_error=myisamchk(&check_param, *(argv++)); + if ((check_param.testflag & T_REP_ANY) != T_REP) + check_param.testflag&= ~T_REP; + (void) fflush(stdout); + (void) fflush(stderr); + if ((check_param.error_printed | check_param.warning_printed) && + (check_param.testflag & T_FORCE_CREATE) && + (!(check_param.testflag & (T_REP_ANY | T_SORT_RECORDS | T_SORT_INDEX)))) + { + ulonglong old_testflag=check_param.testflag; + if (!(check_param.testflag & T_REP_ANY)) + check_param.testflag|= T_REP_BY_SORT; + check_param.testflag&= ~T_EXTEND; /* Don't needed */ + error|=myisamchk(&check_param, argv[-1]); + check_param.testflag= old_testflag; + (void) fflush(stdout); + (void) fflush(stderr); + } + else + error|=new_error; + if (argc && (!(check_param.testflag & T_SILENT) || check_param.testflag & T_INFO)) + { + puts("\n---------\n"); + (void) fflush(stdout); + } + } + if (check_param.total_files > 1) + { /* Only if descript */ + char buff[22],buff2[22]; + if (!(check_param.testflag & T_SILENT) || check_param.testflag & T_INFO) + puts("\n---------\n"); + printf("\nTotal of all %d MyISAM-files:\nData records: %9s Deleted blocks: %9s\n",check_param.total_files,llstr(check_param.total_records,buff), + llstr(check_param.total_deleted,buff2)); + } + free_defaults(default_argv); + free_tmpdir(&myisamchk_tmpdir); + ft_free_stopwords(); + my_end(check_param.testflag & T_INFO ? MY_CHECK_ERROR | MY_GIVE_INFO : MY_CHECK_ERROR); + rc= (uchar) error; + exit(rc); +#ifndef _lint + return 0; /* No compiler warning */ +#endif +} /* main */ + +enum options_mc { + OPT_CHARSETS_DIR=256, OPT_SET_COLLATION,OPT_START_CHECK_POS, + OPT_CORRECT_CHECKSUM, OPT_CREATE_MISSING_KEYS, OPT_KEY_BUFFER_SIZE, + OPT_KEY_CACHE_BLOCK_SIZE, OPT_MYISAM_BLOCK_SIZE, + OPT_READ_BUFFER_SIZE, OPT_WRITE_BUFFER_SIZE, OPT_SORT_BUFFER_SIZE, + OPT_SORT_KEY_BLOCKS, OPT_DECODE_BITS, OPT_FT_MIN_WORD_LEN, + OPT_FT_MAX_WORD_LEN, OPT_FT_STOPWORD_FILE, + OPT_MAX_RECORD_LENGTH, OPT_STATS_METHOD +}; + +static struct my_option my_long_options[] = +{ + {"analyze", 'a', + "Analyze distribution of keys. Will make some joins in MySQL faster. You can check the calculated distribution.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"block-search", 'b', + "No help available.", + 0, 0, 0, GET_ULONG, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"backup", 'B', + "Make a backup of the .MYD file as 'filename-time.BAK'.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"character-sets-dir", OPT_CHARSETS_DIR, + "Directory where character sets are.", + (char**) &charsets_dir, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"check", 'c', + "Check table for errors.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"check-only-changed", 'C', + "Check only tables that have changed since last check. It also applies to other requested actions (e.g. --analyze will be ignored if the table is already analyzed).", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"correct-checksum", OPT_CORRECT_CHECKSUM, + "Correct checksum information for table.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"create-missing-keys", OPT_CREATE_MISSING_KEYS, + "Create missing keys. This assumes that the data file is correct and that " + "the the number of rows stored in the index file is correct. Enables " + "--quick", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, +#ifndef DBUG_OFF + {"debug", '#', + "Output debug log. Often this is 'd:t:o,filename'.", + 0, 0, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, +#endif + {"description", 'd', + "Prints some information about table.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"data-file-length", 'D', + "Max length of data file (when recreating data-file when it's full).", + &check_param.max_data_file_length, + &check_param.max_data_file_length, + 0, GET_LL, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"extend-check", 'e', + "If used when checking a table, ensure that the table is 100 percent consistent, which will take a long time. If used when repairing a table, try to recover every possible row from the data file. Normally this will also find a lot of garbage rows; Don't use this option with repair if you are not totally desperate.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"fast", 'F', + "Check only tables that haven't been closed properly. It also applies to other requested actions (e.g. --analyze will be ignored if the table is already analyzed).", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"force", 'f', + "Restart with -r if there are any errors in the table. States will be updated as with --update-state.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"HELP", 'H', + "Display this help and exit.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"help", '?', + "Display this help and exit.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"information", 'i', + "Print statistics information about table that is checked.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"keys-used", 'k', + "Tell MyISAM to update only some specific keys. # is a bit mask of which keys to use. This can be used to get faster inserts.", + &check_param.keys_in_use, + &check_param.keys_in_use, + 0, GET_ULL, REQUIRED_ARG, -1, 0, 0, 0, 0, 0}, + {"max-record-length", OPT_MAX_RECORD_LENGTH, + "Skip rows bigger than this if myisamchk can't allocate memory to hold it", + &check_param.max_record_length, + &check_param.max_record_length, + 0, GET_ULL, REQUIRED_ARG, LONGLONG_MAX, 0, LONGLONG_MAX, 0, 0, 0}, + {"medium-check", 'm', + "Faster than extend-check, but only finds 99.99% of all errors. Should be good enough for most cases.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"quick", 'q', "Faster repair by not modifying the data file.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"read-only", 'T', + "Don't mark table as checked.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"recover", 'r', + "Can fix almost anything except unique keys that aren't unique.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"parallel-recover", 'p', + "Same as '-r' but creates all the keys in parallel.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"safe-recover", 'o', + "Uses old recovery method; Slower than '-r' but can handle a couple of cases where '-r' reports that it can't fix the data file.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"sort-recover", 'n', + "Force recovering with sorting even if the temporary file was very big.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, +#ifdef DEBUG + {"start-check-pos", OPT_START_CHECK_POS, + "No help available.", + 0, 0, 0, GET_ULL, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, +#endif + {"set-auto-increment", 'A', + "Force auto_increment to start at this or higher value. If no value is given, then sets the next auto_increment value to the highest used value for the auto key + 1.", + &check_param.auto_increment_value, + &check_param.auto_increment_value, + 0, GET_ULL, OPT_ARG, 0, 0, 0, 0, 0, 0}, + {"set-collation", OPT_SET_COLLATION, + "Change the collation used by the index", + &set_collation_name, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"silent", 's', + "Only print errors. One can use two -s to make myisamchk very silent.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"sort-index", 'S', + "Sort index blocks. This speeds up 'read-next' in applications.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"sort-records", 'R', + "Sort records according to an index. This makes your data much more localized and may speed up things. (It may be VERY slow to do a sort the first time!)", + &check_param.opt_sort_key, + &check_param.opt_sort_key, + 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"tmpdir", 't', + "Path for temporary files.", (char**) &opt_tmpdir, + 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"update-state", 'U', + "Mark tables as crashed if any errors were found.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"unpack", 'u', + "Unpack file packed with myisampack.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"verbose", 'v', + "Print more information. This can be used with --description and --check. Use many -v for more verbosity!", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"version", 'V', + "Print version and exit.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"wait", 'w', + "Wait if table is locked.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + { "key_buffer_size", OPT_KEY_BUFFER_SIZE, "", + &check_param.use_buffers, &check_param.use_buffers, 0, + GET_ULL, REQUIRED_ARG, KEY_BUFFER_INIT, MALLOC_OVERHEAD, + SIZE_T_MAX, MALLOC_OVERHEAD, IO_SIZE, 0}, + { "key_cache_block_size", OPT_KEY_CACHE_BLOCK_SIZE, "", + &opt_key_cache_block_size, + &opt_key_cache_block_size, 0, + GET_LONG, REQUIRED_ARG, MI_KEY_BLOCK_LENGTH, MI_MIN_KEY_BLOCK_LENGTH, + MI_MAX_KEY_BLOCK_LENGTH, 0, MI_MIN_KEY_BLOCK_LENGTH, 0}, + { "myisam_block_size", OPT_MYISAM_BLOCK_SIZE, "", + &opt_myisam_block_size, &opt_myisam_block_size, 0, + GET_LONG, REQUIRED_ARG, MI_KEY_BLOCK_LENGTH, MI_MIN_KEY_BLOCK_LENGTH, + MI_MAX_KEY_BLOCK_LENGTH, 0, MI_MIN_KEY_BLOCK_LENGTH, 0}, + { "read_buffer_size", OPT_READ_BUFFER_SIZE, "", + &check_param.read_buffer_length, + &check_param.read_buffer_length, 0, GET_ULONG, REQUIRED_ARG, + READ_BUFFER_INIT, MALLOC_OVERHEAD, + INT_MAX32, MALLOC_OVERHEAD, 1L, 0}, + { "write_buffer_size", OPT_WRITE_BUFFER_SIZE, "", + &check_param.write_buffer_length, + &check_param.write_buffer_length, 0, GET_ULONG, REQUIRED_ARG, + READ_BUFFER_INIT, MALLOC_OVERHEAD, + INT_MAX32, MALLOC_OVERHEAD, 1L, 0}, + { "sort_buffer_size", OPT_SORT_BUFFER_SIZE, + "Deprecated. myisam_sort_buffer_size alias is being used", + &check_param.sort_buffer_length, + &check_param.sort_buffer_length, 0, GET_ULL, REQUIRED_ARG, + SORT_BUFFER_INIT, MIN_SORT_BUFFER + MALLOC_OVERHEAD, + SIZE_T_MAX, MALLOC_OVERHEAD, 1L, 0}, + { "myisam_sort_buffer_size", OPT_SORT_BUFFER_SIZE, + "Alias of sort_buffer_size parameter", + &check_param.sort_buffer_length, + &check_param.sort_buffer_length, 0, GET_ULL, REQUIRED_ARG, + SORT_BUFFER_INIT, MIN_SORT_BUFFER + MALLOC_OVERHEAD, + SIZE_T_MAX, MALLOC_OVERHEAD, 1L, 0}, + { "sort_key_blocks", OPT_SORT_KEY_BLOCKS, "", + &check_param.sort_key_blocks, + &check_param.sort_key_blocks, 0, GET_ULONG, REQUIRED_ARG, + BUFFERS_WHEN_SORTING, 4L, 100L, 0L, 1L, 0}, + { "decode_bits", OPT_DECODE_BITS, "", &decode_bits, + &decode_bits, 0, GET_UINT, REQUIRED_ARG, 9L, 4L, 17L, 0L, 1L, 0}, + { "ft_min_word_len", OPT_FT_MIN_WORD_LEN, "", &ft_min_word_len, + &ft_min_word_len, 0, GET_ULONG, REQUIRED_ARG, 4, 1, HA_FT_MAXCHARLEN, + 0, 1, 0}, + { "ft_max_word_len", OPT_FT_MAX_WORD_LEN, "", &ft_max_word_len, + &ft_max_word_len, 0, GET_ULONG, REQUIRED_ARG, HA_FT_MAXCHARLEN, 10, + HA_FT_MAXCHARLEN, 0, 1, 0}, + { "ft_stopword_file", OPT_FT_STOPWORD_FILE, + "Use stopwords from this file instead of built-in list.", + (char**) &ft_stopword_file, (char**) &ft_stopword_file, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"stats_method", OPT_STATS_METHOD, + "Specifies how index statistics collection code should treat NULLs. " + "Possible values of name are \"nulls_unequal\" (default behavior for 4.1/5.0), " + "\"nulls_equal\" (emulate 4.0 behavior), and \"nulls_ignored\".", + (char**) &myisam_stats_method_str, (char**) &myisam_stats_method_str, 0, + GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + + +static void print_version(void) +{ + printf("%s Ver 2.7 for %s at %s\n", my_progname, SYSTEM_TYPE, + MACHINE_TYPE); +} + + +static void usage(void) +{ + print_version(); + puts("By Monty, for your professional use"); + puts("This software comes with NO WARRANTY: see the PUBLIC for details.\n"); + puts("Description, check and repair of MyISAM tables."); + puts("Used without options all tables on the command will be checked for errors"); + printf("Usage: %s [OPTIONS] tables[.MYI]\n", my_progname_short); + printf("\nGlobal options:\n"); +#ifndef DBUG_OFF + printf("\ + -#, --debug=... Output debug log. Often this is 'd:t:o,filename'.\n"); +#endif + printf("\ + -H, --HELP Display this help and exit.\n\ + -?, --help Display this help and exit.\n\ + -t, --tmpdir=path Path for temporary files. Multiple paths can be\n\ + specified, separated by "); +#if defined( _WIN32) + printf("semicolon (;)"); +#else + printf("colon (:)"); +#endif + printf(", they will be used\n\ + in a round-robin fashion.\n\ + -s, --silent Only print errors. One can use two -s to make\n\ + myisamchk very silent.\n\ + -v, --verbose Print more information. This can be used with\n\ + --description and --check. Use many -v for more verbosity.\n\ + -V, --version Print version and exit.\n\ + -w, --wait Wait if table is locked.\n\n"); +#ifdef DEBUG + puts(" --start-check-pos=# Start reading file at given offset.\n"); +#endif + + puts("Check options (check is the default action for myisamchk):\n\ + -c, --check Check table for errors.\n\ + -e, --extend-check Check the table VERY thoroughly. Only use this in\n\ + extreme cases as myisamchk should normally be able to\n\ + find out if the table is ok even without this switch.\n\ + -F, --fast Check only tables that haven't been closed properly.\n\ + -C, --check-only-changed\n\ + Check only tables that have changed since last check.\n\ + -f, --force Restart with '-r' if there are any errors in the table.\n\ + States will be updated as with '--update-state'.\n\ + -i, --information Print statistics information about table that is checked.\n\ + -m, --medium-check Faster than extend-check, but only finds 99.99% of\n\ + all errors. Should be good enough for most cases.\n\ + -U --update-state Mark tables as crashed if you find any errors.\n\ + -T, --read-only Don't mark table as checked.\n"); + + puts("Repair options (When using '-r' or '-o'):\n\ + -B, --backup Make a backup of the .MYD file as 'filename-time.BAK'.\n\ + --correct-checksum Correct checksum information for table.\n\ + -D, --data-file-length=# Max length of data file (when recreating data\n\ + file when it's full).\n\ + -e, --extend-check Try to recover every possible row from the data file\n\ + Normally this will also find a lot of garbage rows;\n\ + Don't use this option if you are not totally desperate.\n\ + -f, --force Overwrite old temporary files. Add another --force to\n\ + avoid 'myisam_sort_buffer_size is too small' errors.\n\ + In this case we will attempt to do the repair with the\n\ + given myisam_sort_buffer_size and dynamically allocate\n\ + as many management buffers as needed.\n\ + -k, --keys-used=# Tell MyISAM to update only some specific keys. # is a\n\ + bit mask of which keys to use. This can be used to\n\ + get faster inserts.\n\ + --create-missing-keys\n\ + Create missing keys. This assumes that the data\n\ + file is correct and that the the number of rows stored\n\ + in the index file is correct. Enables --quick\n\ + --max-record-length=#\n\ + Skip rows bigger than this if myisamchk can't allocate\n\ + memory to hold it.\n\ + -r, --recover Can fix almost anything except unique keys that aren't\n\ + unique.\n\ + -n, --sort-recover Forces recovering with sorting even if the temporary\n\ + file would be very big.\n\ + -p, --parallel-recover\n\ + Uses the same technique as '-r' and '-n', but creates\n\ + all the keys in parallel, in different threads.\n\ + -o, --safe-recover Uses old recovery method; Slower than '-r' but can\n\ + handle a couple of cases where '-r' reports that it\n\ + can't fix the data file.\n\ + --character-sets-dir=...\n\ + Directory where character sets are.\n\ + --set-collation=name\n\ + Change the collation used by the index.\n\ + -q, --quick Faster repair by not modifying the data file.\n\ + One can give a second '-q' to force myisamchk to\n\ + modify the original datafile in case of duplicate keys.\n\ + NOTE: Tables where the data file is corrupted can't be\n\ + fixed with this option.\n\ + -u, --unpack Unpack file packed with myisampack.\n\ +"); + + puts("Other actions:\n\ + -a, --analyze Analyze distribution of keys. Will make some joins in\n\ + MySQL faster. You can check the calculated distribution\n\ + by using '--description --verbose table_name'.\n\ + --stats_method=name Specifies how index statistics collection code should\n\ + treat NULLs. Possible values of name are \"nulls_unequal\"\n\ + (default for 4.1/5.0), \"nulls_equal\" (emulate 4.0), and \n\ + \"nulls_ignored\".\n\ + -d, --description Prints some information about table.\n\ + -A, --set-auto-increment[=value]\n\ + Force auto_increment to start at this or higher value\n\ + If no value is given, then sets the next auto_increment\n\ + value to the highest used value for the auto key + 1.\n\ + -S, --sort-index Sort index blocks. This speeds up 'read-next' in\n\ + applications.\n\ + -R, --sort-records=#\n\ + Sort records according to an index. This makes your\n\ + data much more localized and may speed up things\n\ + (It may be VERY slow to do a sort the first time!).\n\ + -b, --block-search=#\n\ + Find a record, a block at given offset belongs to."); + + print_defaults("my", load_default_groups); + my_print_variables(my_long_options); +} + + +const char *myisam_stats_method_names[] = {"nulls_unequal", "nulls_equal", + "nulls_ignored", NullS}; +TYPELIB myisam_stats_method_typelib= { + array_elements(myisam_stats_method_names) - 1, "", + myisam_stats_method_names, NULL}; + + /* Read options */ + +static my_bool +get_one_option(const struct my_option *opt, + const char *argument, const char *filename __attribute__((unused))) +{ + switch (opt->id) { + case 'a': + if (argument == disabled_my_option) + check_param.testflag&= ~T_STATISTICS; + else + check_param.testflag|= T_STATISTICS; + break; + case 'A': + if (argument) + check_param.auto_increment_value= strtoull(argument, NULL, 0); + else + check_param.auto_increment_value= 0; /* Set to max used value */ + check_param.testflag|= T_AUTO_INC; + break; + case 'b': + check_param.search_after_block= strtoul(argument, NULL, 10); + break; + case 'B': + if (argument == disabled_my_option) + check_param.testflag&= ~T_BACKUP_DATA; + else + check_param.testflag|= T_BACKUP_DATA; + break; + case 'c': + if (argument == disabled_my_option) + check_param.testflag&= ~T_CHECK; + else + check_param.testflag|= T_CHECK; + break; + case 'C': + if (argument == disabled_my_option) + check_param.testflag&= ~(T_CHECK | T_CHECK_ONLY_CHANGED); + else + check_param.testflag|= T_CHECK | T_CHECK_ONLY_CHANGED; + break; + case 'D': + check_param.max_data_file_length=strtoll(argument, NULL, 10); + break; + case 's': /* silent */ + if (argument == disabled_my_option) + check_param.testflag&= ~(T_SILENT | T_VERY_SILENT); + else + { + if (check_param.testflag & T_SILENT) + check_param.testflag|= T_VERY_SILENT; + check_param.testflag|= T_SILENT; + check_param.testflag&= ~T_WRITE_LOOP; + } + break; + case 'w': + if (argument == disabled_my_option) + check_param.testflag&= ~T_WAIT_FOREVER; + else + check_param.testflag|= T_WAIT_FOREVER; + break; + case 'd': /* description if isam-file */ + if (argument == disabled_my_option) + check_param.testflag&= ~T_DESCRIPT; + else + check_param.testflag|= T_DESCRIPT; + break; + case 'e': /* extend check */ + if (argument == disabled_my_option) + check_param.testflag&= ~T_EXTEND; + else + check_param.testflag|= T_EXTEND; + break; + case 'i': + if (argument == disabled_my_option) + check_param.testflag&= ~T_INFO; + else + check_param.testflag|= T_INFO; + break; + case 'f': + if (argument == disabled_my_option) + { + check_param.tmpfile_createflag= O_RDWR | O_TRUNC | O_EXCL; + check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE | + T_FORCE_SORT_MEMORY); + } + else + { + if (check_param.testflag & T_FORCE_CREATE) + check_param.testflag= T_FORCE_SORT_MEMORY; + check_param.tmpfile_createflag= O_RDWR | O_TRUNC; + check_param.testflag|= T_FORCE_CREATE | T_UPDATE_STATE; + } + break; + case 'F': + if (argument == disabled_my_option) + check_param.testflag&= ~T_FAST; + else + check_param.testflag|= T_FAST; + break; + case 'k': + check_param.keys_in_use= (ulonglong) strtoll(argument, NULL, 10); + break; + case 'm': + if (argument == disabled_my_option) + check_param.testflag&= ~T_MEDIUM; + else + check_param.testflag|= T_MEDIUM; /* Medium check */ + break; + case 'r': /* Repair table */ + check_param.testflag&= ~T_REP_ANY; + if (argument != disabled_my_option) + check_param.testflag|= T_REP_BY_SORT; + break; + case 'p': + check_param.testflag&= ~T_REP_ANY; + if (argument != disabled_my_option) + check_param.testflag|= T_REP_PARALLEL; + break; + case 'o': + check_param.testflag&= ~T_REP_ANY; + check_param.force_sort= 0; + if (argument != disabled_my_option) + { + check_param.testflag|= T_REP; + my_disable_async_io= 1; /* More safety */ + } + break; + case 'n': + check_param.testflag&= ~T_REP_ANY; + if (argument == disabled_my_option) + check_param.force_sort= 0; + else + { + check_param.testflag|= T_REP_BY_SORT; + check_param.force_sort= 1; + } + break; + case 'q': + if (argument == disabled_my_option) + check_param.testflag&= ~(T_QUICK | T_FORCE_UNIQUENESS); + else + { + /* + If T_QUICK was specified before, but not OPT_CREATE_MISSING_KEYS, + then add T_FORCE_UNIQUENESS. + */ + check_param.testflag|= + ((check_param.testflag & (T_QUICK | T_CREATE_MISSING_KEYS)) == + T_QUICK ? T_FORCE_UNIQUENESS : T_QUICK); + } + break; + case OPT_CREATE_MISSING_KEYS: + if (argument == disabled_my_option) + check_param.testflag&= ~(T_QUICK | T_CREATE_MISSING_KEYS); + else + { + check_param.testflag|= T_QUICK | T_CREATE_MISSING_KEYS; + /* Use repair by sort by default */ + if (!(check_param.testflag & T_REP_ANY)) + check_param.testflag|= T_REP_BY_SORT; + } + break; + case 'u': + if (argument == disabled_my_option) + check_param.testflag&= ~(T_UNPACK | T_REP_BY_SORT); + else + check_param.testflag|= T_UNPACK | T_REP_BY_SORT; + break; + case 'v': /* Verbose */ + if (argument == disabled_my_option) + { + check_param.testflag&= ~T_VERBOSE; + check_param.verbose=0; + } + else + { + check_param.testflag|= T_VERBOSE; + check_param.verbose++; + } + break; + case 'R': /* Sort records */ + if (argument == disabled_my_option) + check_param.testflag&= ~T_SORT_RECORDS; + else + { + check_param.testflag|= T_SORT_RECORDS; + check_param.opt_sort_key= (uint) atoi(argument) - 1; + if (check_param.opt_sort_key >= MI_MAX_KEY) + { + fprintf(stderr, + "The value of the sort key is bigger than max key: %d.\n", + MI_MAX_KEY); + exit(1); + } + } + break; + case 'S': /* Sort index */ + if (argument == disabled_my_option) + check_param.testflag&= ~T_SORT_INDEX; + else + check_param.testflag|= T_SORT_INDEX; + break; + case 'T': + if (argument == disabled_my_option) + check_param.testflag&= ~T_READONLY; + else + check_param.testflag|= T_READONLY; + break; + case 'U': + if (argument == disabled_my_option) + check_param.testflag&= ~T_UPDATE_STATE; + else + check_param.testflag|= T_UPDATE_STATE; + break; + case '#': + if (argument == disabled_my_option) + { + DBUG_POP(); + } + else + { + DBUG_PUSH(argument ? argument : "d:t:o,/tmp/myisamchk.trace"); + } + break; + case 'V': + print_version(); + exit(0); + case OPT_CORRECT_CHECKSUM: + if (argument == disabled_my_option) + check_param.testflag&= ~T_CALC_CHECKSUM; + else + check_param.testflag|= T_CALC_CHECKSUM; + break; + case OPT_STATS_METHOD: + { + int method; + enum_handler_stats_method UNINIT_VAR(method_conv); + + myisam_stats_method_str= argument; + if ((method= find_type(argument, &myisam_stats_method_typelib, + FIND_TYPE_BASIC)) <= 0) + { + fprintf(stderr, "Invalid value of stats_method: %s.\n", argument); + exit(1); + } + switch (method-1) { + case 0: + method_conv= MI_STATS_METHOD_NULLS_EQUAL; + break; + case 1: + method_conv= MI_STATS_METHOD_NULLS_NOT_EQUAL; + break; + case 2: + method_conv= MI_STATS_METHOD_IGNORE_NULLS; + break; + default: abort(); /* Impossible */ + } + check_param.stats_method= method_conv; + break; + } +#ifdef DEBUG /* Only useful if debugging */ + case OPT_START_CHECK_POS: + check_param.start_check_pos= strtoull(argument, NULL, 0); + break; +#endif + case 'H': + my_print_help(my_long_options); + exit(0); + case '?': + usage(); + exit(0); + } + return 0; +} + + +static void get_options(register int *argc,register char ***argv) +{ + int ho_error; + + load_defaults_or_exit("my", load_default_groups, argc, argv); + default_argv= *argv; + if (isatty(fileno(stdout))) + check_param.testflag|=T_WRITE_LOOP; + + if ((ho_error=handle_options(argc, argv, my_long_options, get_one_option))) + exit(ho_error); + + /* If using repair, then update checksum if one uses --update-state */ + if ((check_param.testflag & T_UPDATE_STATE) && + (check_param.testflag & T_REP_ANY)) + check_param.testflag|= T_CALC_CHECKSUM; + + if (*argc == 0) + { + usage(); + exit(-1); + } + + if ((check_param.testflag & T_UNPACK) && + (check_param.testflag & (T_QUICK | T_SORT_RECORDS))) + { + (void) fprintf(stderr, + "%s: --unpack can't be used with --quick or --sort-records\n", + my_progname_short); + exit(1); + } + if ((check_param.testflag & T_READONLY) && + (check_param.testflag & + (T_REP_ANY | T_STATISTICS | T_AUTO_INC | + T_SORT_RECORDS | T_SORT_INDEX | T_FORCE_CREATE))) + { + (void) fprintf(stderr, + "%s: Can't use --readonly when repairing or sorting\n", + my_progname_short); + exit(1); + } + + if (init_tmpdir(&myisamchk_tmpdir, opt_tmpdir)) + exit(1); + + check_param.tmpdir=&myisamchk_tmpdir; + check_param.key_cache_block_size= opt_key_cache_block_size; + + if (set_collation_name) + if (!(set_collation= get_charset_by_name(set_collation_name, + MYF(MY_UTF8_IS_UTF8MB3 | MY_WME)))) + exit(1); + + myisam_block_size=(uint) 1 << my_bit_log2_uint64(opt_myisam_block_size); + return; +} /* get options */ + + + /* Check table */ + +static int myisamchk(HA_CHECK *param, char * filename) +{ + int error,lock_type,recreate; + int rep_quick= MY_TEST(param->testflag & (T_QUICK | T_FORCE_UNIQUENESS)); + MI_INFO *info; + File datafile; + char llbuff[22],llbuff2[22]; + my_bool state_updated=0; + MYISAM_SHARE *share; + int open_mode; + uint open_flags= HA_OPEN_FOR_REPAIR; + DBUG_ENTER("myisamchk"); + + param->out_flag=error=param->warning_printed=param->error_printed= + recreate=0; + datafile=0; + param->isam_file_name=filename; /* For error messages */ + open_mode= param->testflag & (T_DESCRIPT | T_READONLY) ? O_RDONLY : O_RDWR; + if (param->testflag & T_WAIT_FOREVER) + open_flags|= HA_OPEN_WAIT_IF_LOCKED; + else if (param->testflag & T_DESCRIPT) + open_flags|= HA_OPEN_IGNORE_IF_LOCKED | HA_OPEN_FROM_SQL_LAYER; + else + open_flags|= HA_OPEN_ABORT_IF_LOCKED; + if (!(info=mi_open(filename, open_mode, open_flags))) + { + /* Avoid twice printing of isam file name */ + param->error_printed=1; + switch (my_errno) { + case HA_ERR_CRASHED: + mi_check_print_error(param,"'%s' doesn't have a correct index definition. You need to recreate it before you can do a repair",filename); + break; + case HA_ERR_NOT_A_TABLE: + mi_check_print_error(param,"'%s' is not a MyISAM-table",filename); + break; + case HA_ERR_CRASHED_ON_USAGE: + mi_check_print_error(param,"'%s' is marked as crashed",filename); + break; + case HA_ERR_CRASHED_ON_REPAIR: + mi_check_print_error(param,"'%s' is marked as crashed after last repair",filename); + break; + case HA_ERR_OLD_FILE: + mi_check_print_error(param,"'%s' is an old type of MyISAM-table", filename); + break; + case HA_ERR_END_OF_FILE: + mi_check_print_error(param,"Couldn't read complete header from '%s'", filename); + break; + case EAGAIN: + mi_check_print_error(param,"'%s' is locked. Use -w to wait until unlocked",filename); + break; + case ENOENT: + mi_check_print_error(param,"File '%s' doesn't exist",filename); + break; + case EACCES: + mi_check_print_error(param,"You don't have permission to use '%s'",filename); + break; + default: + mi_check_print_error(param,"%d when opening MyISAM-table '%s'", + my_errno,filename); + break; + } + DBUG_RETURN(1); + } + share=info->s; + share->options&= ~HA_OPTION_READ_ONLY_DATA; /* We are modifing it */ + share->tot_locks-= share->r_locks; + share->r_locks=0; + + /* + Skip the checking of the file if: + We are using --fast and the table is closed properly + We are using --check-only-changed-tables and the table hasn't changed + */ + if (param->testflag & (T_FAST | T_CHECK_ONLY_CHANGED)) + { + my_bool need_to_check= mi_is_crashed(info) || share->state.open_count != 0; + + if ((param->testflag & (T_REP_ANY | T_SORT_RECORDS)) && + ((share->state.changed & (STATE_CHANGED | STATE_CRASHED | + STATE_CRASHED_ON_REPAIR) || + !(param->testflag & T_CHECK_ONLY_CHANGED)))) + need_to_check=1; + + if (info->s->base.keys && info->state->records) + { + if ((param->testflag & T_STATISTICS) && + (share->state.changed & STATE_NOT_ANALYZED)) + need_to_check=1; + if ((param->testflag & T_SORT_INDEX) && + (share->state.changed & STATE_NOT_SORTED_PAGES)) + need_to_check=1; + if ((param->testflag & T_REP_BY_SORT) && + (share->state.changed & STATE_NOT_OPTIMIZED_KEYS)) + need_to_check=1; + } + if ((param->testflag & T_CHECK_ONLY_CHANGED) && + (share->state.changed & (STATE_CHANGED | STATE_CRASHED | + STATE_CRASHED_ON_REPAIR))) + need_to_check=1; + if (!need_to_check) + { + if (!(param->testflag & T_SILENT) || param->testflag & T_INFO) + printf("MyISAM file: %s is already checked\n",filename); + if (mi_close(info)) + { + mi_check_print_error(param,"%d when closing MyISAM-table '%s'", + my_errno,filename); + DBUG_RETURN(1); + } + DBUG_RETURN(0); + } + } + if ((param->testflag & (T_REP_ANY | T_STATISTICS | + T_SORT_RECORDS | T_SORT_INDEX)) && + (((param->testflag & T_UNPACK) && + share->data_file_type == COMPRESSED_RECORD) || + mi_uint2korr(share->state.header.state_info_length) != + MI_STATE_INFO_SIZE || + mi_uint2korr(share->state.header.base_info_length) != + MI_BASE_INFO_SIZE || + mi_is_any_intersect_keys_active(param->keys_in_use, share->base.keys, + ~share->state.key_map) || + test_if_almost_full(info) || + info->s->state.header.file_version[3] != myisam_file_magic[3] || + (set_collation && + set_collation->number != share->state.header.language) || + myisam_block_size != MI_KEY_BLOCK_LENGTH)) + { + if (set_collation) + param->language= set_collation->number; + if (recreate_table(param, &info,filename)) + { + (void) fprintf(stderr, + "MyISAM-table '%s' is not fixed because of errors\n", + filename); + return(-1); + } + recreate=1; + if (!(param->testflag & T_REP_ANY)) + { + param->testflag|=T_REP_BY_SORT; /* if only STATISTICS */ + if (!(param->testflag & T_SILENT)) + printf("- '%s' has old table-format. Recreating index\n",filename); + rep_quick= 1; + } + share=info->s; + share->tot_locks-= share->r_locks; + share->r_locks=0; + } + + if (param->testflag & T_DESCRIPT) + { + param->total_files++; + param->total_records+=info->state->records; + param->total_deleted+=info->state->del; + descript(param, info, filename); + } + else + { + if (!stopwords_inited++) + ft_init_stopwords(); + + if (!(param->testflag & T_READONLY)) + lock_type = F_WRLCK; /* table is changed */ + else + lock_type= F_RDLCK; + if (info->lock_type == F_RDLCK) + info->lock_type=F_UNLCK; /* Read only table */ + if (_mi_readinfo(info,lock_type,0)) + { + mi_check_print_error(param,"Can't lock indexfile of '%s', error: %d", + filename,my_errno); + param->error_printed=0; + goto end2; + } + /* + _mi_readinfo() has locked the table. + We mark the table as locked (without doing file locks) to be able to + use functions that only works on locked tables (like row caching). + */ + mi_lock_database(info, F_EXTRA_LCK); + datafile=info->dfile; + + if (param->testflag & (T_REP_ANY | T_SORT_RECORDS | T_SORT_INDEX)) + { + if (param->testflag & T_REP_ANY) + { + ulonglong tmp=share->state.key_map; + mi_copy_keys_active(share->state.key_map, share->base.keys, + param->keys_in_use); + if (tmp != share->state.key_map) + info->update|=HA_STATE_CHANGED; + } + if (rep_quick && chk_del(param, info, param->testflag & ~T_VERBOSE)) + { + if (param->testflag & T_FORCE_CREATE) + { + rep_quick=0; + mi_check_print_info(param,"Creating new data file\n"); + } + else + { + error=1; + mi_check_print_error(param, + "Quick-recover aborted; Run recovery without switch 'q'"); + } + } + if (!error) + { + if ((param->testflag & (T_REP_BY_SORT | T_REP_PARALLEL)) && + (mi_is_any_key_active(share->state.key_map) || + (rep_quick && !param->keys_in_use && !recreate)) && + mi_test_if_sort_rep(info, info->state->records, + info->s->state.key_map, + param->force_sort)) + { + if (param->testflag & T_REP_BY_SORT) + error=mi_repair_by_sort(param,info,filename,rep_quick); + else + error=mi_repair_parallel(param,info,filename,rep_quick); + state_updated=1; + } + else if (param->testflag & T_REP_ANY) + error=mi_repair(param, info,filename,rep_quick); + } + if (!error && param->testflag & T_SORT_RECORDS) + { + /* + The data file is nowadays reopened in the repair code so we should + soon remove the following reopen-code + */ +#ifndef TO_BE_REMOVED + if (param->out_flag & O_NEW_DATA) + { /* Change temp file to org file */ + (void) mysql_file_close(info->dfile, + MYF(MY_WME)); /* Close new file */ + error|=change_to_newfile(filename, MI_NAME_DEXT, DATA_TMP_EXT, + 0, MYF(0)); + if (mi_open_datafile(info, info->s)) + error=1; + param->out_flag&= ~O_NEW_DATA; /* We are using new datafile */ + param->read_cache.file=info->dfile; + } +#endif + if (! error) + { + uint key; + /* + We can't update the index in mi_sort_records if we have a + prefix compressed or fulltext index + */ + my_bool update_index=1; + for (key=0 ; key < share->base.keys; key++) + if (share->keyinfo[key].flag & (HA_BINARY_PACK_KEY|HA_FULLTEXT)) + update_index=0; + + error=mi_sort_records(param,info,filename,param->opt_sort_key, + /* what is the following parameter for ? */ + (my_bool) !(param->testflag & T_REP_ANY), + update_index); + datafile=info->dfile; /* This is now locked */ + if (!error && !update_index) + { + if (param->verbose) + puts("Table had a compressed index; We must now recreate the index"); + error=mi_repair_by_sort(param,info,filename,1); + } + } + } + if (!error && param->testflag & T_SORT_INDEX) + error=mi_sort_index(param,info,filename); + if (!error) + share->state.changed&= ~(STATE_CHANGED | STATE_CRASHED | + STATE_CRASHED_ON_REPAIR); + else + mi_mark_crashed(info); + } + else if ((param->testflag & T_CHECK) || !(param->testflag & T_AUTO_INC)) + { + if (!(param->testflag & T_SILENT) || param->testflag & T_INFO) + printf("Checking MyISAM file: %s\n",filename); + if (!(param->testflag & T_SILENT)) + printf("Data records: %7s Deleted blocks: %7s\n", + llstr(info->state->records,llbuff), + llstr(info->state->del,llbuff2)); + error =chk_status(param,info); + mi_intersect_keys_active(share->state.key_map, param->keys_in_use); + error =chk_size(param,info); + if (!error || !(param->testflag & (T_FAST | T_FORCE_CREATE))) + error|=chk_del(param, info,param->testflag); + if ((!error || (!(param->testflag & (T_FAST | T_FORCE_CREATE)) && + !param->start_check_pos))) + { + error|=chk_key(param, info); + if (!error && (param->testflag & (T_STATISTICS | T_AUTO_INC))) + error=update_state_info(param, info, + ((param->testflag & T_STATISTICS) ? + UPDATE_STAT : 0) | + ((param->testflag & T_AUTO_INC) ? + UPDATE_AUTO_INC : 0)); + } + if ((!rep_quick && !error) || + !(param->testflag & (T_FAST | T_FORCE_CREATE))) + { + if (param->testflag & (T_EXTEND | T_MEDIUM)) + (void) init_key_cache(dflt_key_cache,opt_key_cache_block_size, + (size_t)param->use_buffers, 0, 0, 0, 0); + (void) init_io_cache(¶m->read_cache,datafile, + (uint) param->read_buffer_length, + READ_CACHE, + (param->start_check_pos ? + param->start_check_pos : + share->pack.header_length), + 1, + MYF(MY_WME)); + lock_memory(param); + if ((info->s->options & (HA_OPTION_PACK_RECORD | + HA_OPTION_COMPRESS_RECORD)) || + (param->testflag & (T_EXTEND | T_MEDIUM))) + error|= chk_data_link(param, info, + MY_TEST(param->testflag & T_EXTEND)); + error|=flush_blocks(param, share->key_cache, share->kfile, + &share->dirty_part_map); + (void) end_io_cache(¶m->read_cache); + } + if (!error) + { + if ((share->state.changed & STATE_CHANGED) && + (param->testflag & T_UPDATE_STATE)) + info->update|=HA_STATE_CHANGED | HA_STATE_ROW_CHANGED; + share->state.changed&= ~(STATE_CHANGED | STATE_CRASHED | + STATE_CRASHED_ON_REPAIR); + } + else if (!mi_is_crashed(info) && + (param->testflag & T_UPDATE_STATE)) + { /* Mark crashed */ + mi_mark_crashed(info); + info->update|=HA_STATE_CHANGED | HA_STATE_ROW_CHANGED; + } + } + } + if ((param->testflag & T_AUTO_INC) || + ((param->testflag & T_REP_ANY) && info->s->base.auto_key)) + update_auto_increment_key(param, info, + (my_bool) !MY_TEST(param->testflag & T_AUTO_INC)); + + if (!(param->testflag & T_DESCRIPT)) + { + if (info->update & HA_STATE_CHANGED && ! (param->testflag & T_READONLY)) + error|=update_state_info(param, info, + UPDATE_OPEN_COUNT | + (((param->testflag & T_REP_ANY) ? + UPDATE_TIME : 0) | + (state_updated ? UPDATE_STAT : 0) | + ((param->testflag & T_SORT_RECORDS) ? + UPDATE_SORT : 0))); + (void) lock_file(param, share->kfile,0L,F_UNLCK,"indexfile",filename); + info->update&= ~HA_STATE_CHANGED; + } + mi_lock_database(info, F_UNLCK); +end2: + if (mi_close(info)) + { + mi_check_print_error(param,"%d when closing MyISAM-table '%s'",my_errno,filename); + DBUG_RETURN(1); + } + if (error == 0) + { + if (param->out_flag & O_NEW_DATA) + error|=change_to_newfile(filename,MI_NAME_DEXT,DATA_TMP_EXT, + param->backup_time, + ((param->testflag & T_BACKUP_DATA) ? + MYF(MY_REDEL_MAKE_BACKUP) : MYF(0))); + } + (void) fflush(stdout); (void) fflush(stderr); + if (param->error_printed) + { + if (param->testflag & (T_REP_ANY | T_SORT_RECORDS | T_SORT_INDEX)) + { + (void) fprintf(stderr, + "MyISAM-table '%s' is not fixed because of errors\n", + filename); + if (param->testflag & T_REP_ANY) + (void) fprintf(stderr, + "Try fixing it by using the --safe-recover (-o), the --force (-f) option or by not using the --quick (-q) flag\n"); + } + else if (!(param->error_printed & 2) && + !(param->testflag & T_FORCE_CREATE)) + (void) fprintf(stderr, + "MyISAM-table '%s' is corrupted\nFix it using switch \"-r\" or \"-o\"\n", + filename); + } + else if (param->warning_printed && + ! (param->testflag & (T_REP_ANY | T_SORT_RECORDS | T_SORT_INDEX | + T_FORCE_CREATE))) + (void) fprintf(stderr, "MyISAM-table '%s' is usable but should be fixed\n", + filename); + (void) fflush(stderr); + DBUG_RETURN(error); +} /* myisamchk */ + + + /* Write info about table */ + +static void descript(HA_CHECK *param, register MI_INFO *info, char * name) +{ + uint key,keyseg_nr,field,start; + reg3 MI_KEYDEF *keyinfo; + reg2 HA_KEYSEG *keyseg; + reg4 const char *text; + char buff[160],length[10],*pos,*end; + enum en_fieldtype type; + MYISAM_SHARE *share=info->s; + char llbuff[22],llbuff2[22]; + DBUG_ENTER("describe"); + + printf("\nMyISAM file: %s\n",name); + fputs("Record format: ",stdout); + if (share->options & HA_OPTION_COMPRESS_RECORD) + puts("Compressed"); + else if (share->options & HA_OPTION_PACK_RECORD) + puts("Packed"); + else + puts("Fixed length"); + printf("Character set: %s (%d)\n", + get_charset_name(share->state.header.language), + share->state.header.language); + + if (param->testflag & T_VERBOSE) + { + printf("File-version: %d\n", + (int) share->state.header.file_version[3]); + if (share->state.create_time) + { + get_date(buff,1,share->state.create_time); + printf("Creation time: %s\n",buff); + } + if (share->state.check_time) + { + get_date(buff,1,share->state.check_time); + printf("Recover time: %s\n",buff); + } + pos=buff; + if (share->state.changed & STATE_CRASHED) + strmov(buff, share->state.changed & STATE_CRASHED_ON_REPAIR ? + "crashed on repair" : "crashed"); + else + { + if (share->state.open_count) + pos=strmov(pos,"open,"); + if (share->state.changed & STATE_CHANGED) + pos=strmov(pos,"changed,"); + else + pos=strmov(pos,"checked,"); + if (!(share->state.changed & STATE_NOT_ANALYZED)) + pos=strmov(pos,"analyzed,"); + if (!(share->state.changed & STATE_NOT_OPTIMIZED_KEYS)) + pos=strmov(pos,"optimized keys,"); + if (!(share->state.changed & STATE_NOT_SORTED_PAGES)) + pos=strmov(pos,"sorted index pages,"); + pos[-1]=0; /* Remove extra ',' */ + } + printf("Status: %s\n",buff); + if (share->base.auto_key) + { + printf("Auto increment key: %13d Last value: %13s\n", + share->base.auto_key, + llstr(share->state.auto_increment,llbuff)); + } + if (share->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD)) + printf("Checksum: %23s\n",llstr(info->state->checksum,llbuff)); + + if (share->options & HA_OPTION_DELAY_KEY_WRITE) + printf("Keys are only flushed at close\n"); + + } + printf("Data records: %13s Deleted blocks: %13s\n", + llstr(info->state->records,llbuff),llstr(info->state->del,llbuff2)); + if (param->testflag & T_SILENT) + DBUG_VOID_RETURN; /* This is enough */ + + if (param->testflag & T_VERBOSE) + { +#ifdef USE_RELOC + printf("Init-relocation: %13s\n",llstr(share->base.reloc,llbuff)); +#endif + printf("Datafile parts: %13s Deleted data: %13s\n", + llstr(share->state.split,llbuff), + llstr(info->state->empty,llbuff2)); + printf("Datafile pointer (bytes):%9d Keyfile pointer (bytes):%9d\n", + share->rec_reflength,share->base.key_reflength); + printf("Datafile length: %13s Keyfile length: %13s\n", + llstr(info->state->data_file_length,llbuff), + llstr(info->state->key_file_length,llbuff2)); + + if (info->s->base.reloc == 1L && info->s->base.records == 1L) + puts("This is a one-record table"); + else + { + if (share->base.max_data_file_length != HA_OFFSET_ERROR || + share->base.max_key_file_length != HA_OFFSET_ERROR) + printf("Max datafile length: %13s Max keyfile length: %13s\n", + llstr(share->base.max_data_file_length-1,llbuff), + ullstr(share->base.max_key_file_length - 1, llbuff2)); + } + } + + printf("Recordlength: %13d\n",(int) share->base.pack_reclength); + if (! mi_is_all_keys_active(share->state.key_map, share->base.keys)) + { + longlong2str(share->state.key_map,buff,2); + printf("Using only keys '%s' of %d possibly keys\n", + buff, share->base.keys); + } + puts("\ntable description:"); + printf("Key Start Len Index Type"); + if (param->testflag & T_VERBOSE) + printf(" Rec/key Root Blocksize"); + (void) putchar('\n'); + + for (key=keyseg_nr=0, keyinfo= &share->keyinfo[0] ; + key < share->base.keys; + key++,keyinfo++) + { + keyseg=keyinfo->seg; + if (keyinfo->flag & HA_NOSAME) text="unique "; + else if (keyinfo->flag & HA_FULLTEXT) text="fulltext "; + else text="multip."; + + pos=buff; + if (keyseg->flag & HA_REVERSE_SORT) + *pos++ = '-'; + pos=strmov(pos,type_names[keyseg->type]); + *pos++ = ' '; + *pos=0; + if (keyinfo->flag & HA_PACK_KEY) + pos=strmov(pos,prefix_packed_txt); + if (keyinfo->flag & HA_BINARY_PACK_KEY) + pos=strmov(pos,bin_packed_txt); + if (keyseg->flag & HA_SPACE_PACK) + pos=strmov(pos,diff_txt); + if (keyseg->flag & HA_BLOB_PART) + pos=strmov(pos,blob_txt); + if (keyseg->flag & HA_NULL_PART) + pos=strmov(pos,null_txt); + *pos=0; + + printf("%-4d%-6ld%-3d %-8s%-21s", + key+1,(long) keyseg->start+1,keyseg->length,text,buff); + if (share->state.key_root[key] != HA_OFFSET_ERROR) + llstr(share->state.key_root[key],buff); + else + buff[0]=0; + if (param->testflag & T_VERBOSE) + printf("%11lu %12s %10d", + share->state.rec_per_key_part[keyseg_nr++], + buff,keyinfo->block_length); + (void) putchar('\n'); + while ((++keyseg)->type != HA_KEYTYPE_END) + { + pos=buff; + if (keyseg->flag & HA_REVERSE_SORT) + *pos++ = '-'; + pos=strmov(pos,type_names[keyseg->type]); + *pos++= ' '; + if (keyseg->flag & HA_SPACE_PACK) + pos=strmov(pos,diff_txt); + if (keyseg->flag & HA_BLOB_PART) + pos=strmov(pos,blob_txt); + if (keyseg->flag & HA_NULL_PART) + pos=strmov(pos,null_txt); + *pos=0; + printf(" %-6ld%-3d %-21s", + (long) keyseg->start+1,keyseg->length,buff); + if (param->testflag & T_VERBOSE) + printf("%11lu", share->state.rec_per_key_part[keyseg_nr++]); + (void) putchar('\n'); + } + keyseg++; + } + if (share->state.header.uniques) + { + MI_UNIQUEDEF *uniqueinfo; + puts("\nUnique Key Start Len Nullpos Nullbit Type"); + for (key=0,uniqueinfo= &share->uniqueinfo[0] ; + key < share->state.header.uniques; key++, uniqueinfo++) + { + my_bool new_row=0; + char null_bit[8],null_pos[8]; + printf("%-8d%-5d",key+1,uniqueinfo->key+1); + for (keyseg=uniqueinfo->seg ; keyseg->type != HA_KEYTYPE_END ; keyseg++) + { + if (new_row) + fputs(" ",stdout); + null_bit[0]=null_pos[0]=0; + if (keyseg->null_bit) + { + my_snprintf(null_bit, sizeof(null_bit), "%d", keyseg->null_bit); + my_snprintf(null_pos, sizeof(null_pos), "%ld", (long) keyseg->null_pos+1); + } + printf("%-7ld%-5d%-9s%-10s%-30s\n", + (long) keyseg->start+1,keyseg->length, + null_pos,null_bit, + type_names[keyseg->type]); + new_row=1; + } + } + } + if (param->verbose > 1) + { + char null_bit[8],null_pos[8]; + printf("\nField Start Length Nullpos Nullbit Type"); + if (share->options & HA_OPTION_COMPRESS_RECORD) + printf(" Huff tree Bits"); + (void) putchar('\n'); + start=1; + for (field=0 ; field < share->base.fields ; field++) + { + if (share->options & HA_OPTION_COMPRESS_RECORD) + type=share->rec[field].base_type; + else + type=(enum en_fieldtype) share->rec[field].type; + end=strmov(buff,field_pack[type]); + if (end != buff) + { + *(end++)=','; + *(end++)=' '; + } + if (share->options & HA_OPTION_COMPRESS_RECORD) + { + if (share->rec[field].pack_type & PACK_TYPE_SELECTED) + end=strmov(end,"not_always, "); + if (share->rec[field].pack_type & PACK_TYPE_SPACE_FIELDS) + end=strmov(end,"no empty, "); + if (share->rec[field].pack_type & PACK_TYPE_ZERO_FILL) + { + sprintf(end,"zerofill(%d), ",share->rec[field].space_length_bits); + end=strend(end); + } + } + if (end != buff) + end[-2]= 0; /* Remove ", " */ + int10_to_str((long) share->rec[field].length,length,10); + null_bit[0]=null_pos[0]=0; + if (share->rec[field].null_bit) + { + sprintf(null_bit,"%d",share->rec[field].null_bit); + sprintf(null_pos,"%d",share->rec[field].null_pos+1); + } + printf("%-6d%-6d%-7s%-8s%-8s%-35s",field+1,start,length, + null_pos, null_bit, buff); + if (share->options & HA_OPTION_COMPRESS_RECORD) + { + if (share->rec[field].huff_tree) + printf("%3d %2d", + (uint) (share->rec[field].huff_tree-share->decode_trees)+1, + share->rec[field].huff_tree->quick_table_bits); + } + (void) putchar('\n'); + start+=share->rec[field].length; + } + } + DBUG_VOID_RETURN; +} /* describe */ + + + /* Sort records according to one key */ + +static int mi_sort_records(HA_CHECK *param, + register MI_INFO *info, char * name, + uint sort_key, + my_bool write_info, + my_bool update_index) +{ + int got_error; + uint key; + MI_KEYDEF *keyinfo; + File new_file; + uchar *temp_buff; + ha_rows old_record_count; + MYISAM_SHARE *share=info->s; + char llbuff[22],llbuff2[22]; + MI_SORT_INFO sort_info; + MI_SORT_PARAM sort_param; + DBUG_ENTER("sort_records"); + + bzero((char*)&sort_info,sizeof(sort_info)); + bzero((char*)&sort_param,sizeof(sort_param)); + sort_param.sort_info=&sort_info; + sort_info.param=param; + keyinfo= &share->keyinfo[sort_key]; + got_error=1; + temp_buff=0; + new_file= -1; + + if (! mi_is_key_active(share->state.key_map, sort_key)) + { + mi_check_print_warning(param, + "Can't sort table '%s' on key %d; No such key", + name,sort_key+1); + param->error_printed=0; + DBUG_RETURN(0); /* Nothing to do */ + } + if (keyinfo->flag & HA_FULLTEXT) + { + mi_check_print_warning(param,"Can't sort table '%s' on FULLTEXT key %d", + name,sort_key+1); + param->error_printed=0; + DBUG_RETURN(0); /* Nothing to do */ + } + if (share->data_file_type == COMPRESSED_RECORD) + { + mi_check_print_warning(param,"Can't sort read-only table '%s'", name); + param->error_printed=0; + DBUG_RETURN(0); /* Nothing to do */ + } + if (!(param->testflag & T_SILENT)) + { + printf("- Sorting records for MyISAM-table '%s'\n",name); + if (write_info) + printf("Data records: %9s Deleted: %9s\n", + llstr(info->state->records,llbuff), + llstr(info->state->del,llbuff2)); + } + if (share->state.key_root[sort_key] == HA_OFFSET_ERROR) + DBUG_RETURN(0); /* Nothing to do */ + + init_key_cache(dflt_key_cache, opt_key_cache_block_size, + (size_t) param->use_buffers, 0, 0, 0, 0); + if (init_io_cache(&info->rec_cache,-1,(uint) param->write_buffer_length, + WRITE_CACHE,share->pack.header_length,1, + MYF(MY_WME | MY_WAIT_IF_FULL))) + goto err; + info->opt_flag|=WRITE_CACHE_USED; + + if (!(temp_buff=(uchar*) my_alloca((uint) keyinfo->block_length))) + { + mi_check_print_error(param,"Not enough memory for key block"); + goto err; + } + + if (!mi_alloc_rec_buff(info, -1, &sort_param.record)) + { + mi_check_print_error(param,"Not enough memory for record"); + goto err; + } + fn_format(param->temp_filename,name,"", MI_NAME_DEXT,2+4+32); + new_file= mysql_file_create(mi_key_file_datatmp, + fn_format(param->temp_filename, + param->temp_filename, "", + DATA_TMP_EXT, 2+4), + 0, param->tmpfile_createflag, + MYF(0)); + if (new_file < 0) + { + mi_check_print_error(param,"Can't create new tempfile: '%s'", + param->temp_filename); + goto err; + } + if (share->pack.header_length) + if (filecopy(param,new_file,info->dfile,0L,share->pack.header_length, + "datafile-header")) + goto err; + info->rec_cache.file=new_file; /* Use this file for cacheing*/ + + lock_memory(param); + for (key=0 ; key < share->base.keys ; key++) + share->keyinfo[key].flag|= HA_SORT_ALLOWS_SAME; + + if (mysql_file_pread(share->kfile,(uchar*) temp_buff, + (uint) keyinfo->block_length, + share->state.key_root[sort_key], + MYF(MY_NABP+MY_WME))) + { + mi_check_print_error(param,"Can't read indexpage from filepos: %s", + (ulong) share->state.key_root[sort_key]); + goto err; + } + + /* Setup param for sort_write_record */ + sort_info.info=info; + sort_info.new_data_file_type=share->data_file_type; + sort_param.fix_datafile=1; + sort_param.master=1; + sort_param.filepos=share->pack.header_length; + old_record_count=info->state->records; + info->state->records=0; + if (sort_info.new_data_file_type != COMPRESSED_RECORD) + info->state->checksum=0; + + if (sort_record_index(&sort_param,info,keyinfo,share->state.key_root[sort_key], + temp_buff, sort_key,new_file,update_index) || + write_data_suffix(&sort_info,1) || + flush_io_cache(&info->rec_cache)) + goto err; + + if (info->state->records != old_record_count) + { + mi_check_print_error(param,"found %s of %s records", + llstr(info->state->records,llbuff), + llstr(old_record_count,llbuff2)); + goto err; + } + + (void) mysql_file_close(info->dfile,MYF(MY_WME)); + param->out_flag|=O_NEW_DATA; /* Data in new file */ + info->dfile=new_file; /* Use new datafile */ + info->state->del=0; + info->state->empty=0; + share->state.dellink= HA_OFFSET_ERROR; + info->state->data_file_length=sort_param.filepos; + share->state.split=info->state->records; /* Only hole records */ + share->state.version=(ulong) time((time_t*) 0); + + info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + + if (param->testflag & T_WRITE_LOOP) + { + (void) fputs(" \r",stdout); (void) fflush(stdout); + } + got_error=0; + +err: + if (got_error && new_file >= 0) + { + (void) end_io_cache(&info->rec_cache); + (void) mysql_file_close(new_file,MYF(MY_WME)); + (void) mysql_file_delete(mi_key_file_dfile, param->temp_filename, + MYF(MY_WME)); + } + if (temp_buff) + { + my_afree((uchar*) temp_buff); + } + my_free(mi_get_rec_buff_ptr(info, sort_param.record)); + info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED); + (void) end_io_cache(&info->rec_cache); + my_free(sort_info.buff); + sort_info.buff=0; + share->state.sortkey=sort_key; + DBUG_RETURN(flush_blocks(param, share->key_cache, share->kfile, + &share->dirty_part_map) | got_error); +} /* sort_records */ + + + /* Sort records recursive using one index */ + +static int sort_record_index(MI_SORT_PARAM *sort_param,MI_INFO *info, + MI_KEYDEF *keyinfo, + my_off_t page, uchar *buff, uint sort_key, + File new_file,my_bool update_index) +{ + uint nod_flag,used_length,key_length; + uchar *temp_buff,*keypos,*endpos; + my_off_t next_page,rec_pos; + uchar lastkey[HA_MAX_KEY_BUFF]; + char llbuff[22]; + MI_SORT_INFO *sort_info= sort_param->sort_info; + HA_CHECK *param=sort_info->param; + DBUG_ENTER("sort_record_index"); + + nod_flag=mi_test_if_nod(buff); + temp_buff=0; + + if (nod_flag) + { + if (!(temp_buff=(uchar*) my_alloca((uint) keyinfo->block_length))) + { + mi_check_print_error(param,"Not Enough memory"); + DBUG_RETURN(-1); + } + } + used_length=mi_getint(buff); + keypos=buff+2+nod_flag; + endpos=buff+used_length; + for ( ;; ) + { + if (nod_flag) + { + next_page=_mi_kpos(nod_flag,keypos); + if (mysql_file_pread(info->s->kfile,(uchar*) temp_buff, + (uint) keyinfo->block_length, next_page, + MYF(MY_NABP+MY_WME))) + { + mi_check_print_error(param,"Can't read keys from filepos: %s", + llstr(next_page,llbuff)); + goto err; + } + if (sort_record_index(sort_param, info,keyinfo,next_page,temp_buff,sort_key, + new_file, update_index)) + goto err; + } + if (keypos >= endpos || + (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,lastkey)) + == 0) + break; + rec_pos= _mi_dpos(info,0,lastkey+key_length); + + if ((*info->s->read_rnd)(info,sort_param->record,rec_pos,0)) + { + mi_check_print_error(param,"%d when reading datafile",my_errno); + goto err; + } + if (rec_pos != sort_param->filepos && update_index) + { + _mi_dpointer(info,keypos-nod_flag-info->s->rec_reflength, + sort_param->filepos); + if (movepoint(info,sort_param->record,rec_pos,sort_param->filepos, + sort_key)) + { + mi_check_print_error(param,"%d when updating key-pointers",my_errno); + goto err; + } + } + if (sort_write_record(sort_param)) + goto err; + } + /* Clear end of block to get better compression if the table is backuped */ + bzero((uchar*) buff+used_length,keyinfo->block_length-used_length); + if (my_pwrite(info->s->kfile,(uchar*) buff,(uint) keyinfo->block_length, + page,param->myf_rw)) + { + mi_check_print_error(param,"%d when updating keyblock",my_errno); + goto err; + } + if (temp_buff) + my_afree((uchar*) temp_buff); + DBUG_RETURN(0); +err: + if (temp_buff) + my_afree((uchar*) temp_buff); + DBUG_RETURN(1); +} /* sort_record_index */ + + + +/* + Check if myisamchk was killed by a signal + This is overloaded by other programs that want to be able to abort + sorting +*/ + +int killed_ptr(HA_CHECK *param __attribute__((unused))) +{ + return 0; +} + + /* print warnings and errors */ + /* VARARGS */ + +void mi_check_print_info(HA_CHECK *param __attribute__((unused)), + const char *fmt,...) +{ + va_list args; + + param->note_printed=1; + va_start(args,fmt); + (void) vfprintf(stdout, fmt, args); + (void) fputc('\n',stdout); + va_end(args); +} + +/* VARARGS */ + +void mi_check_print_warning(HA_CHECK *param, const char *fmt,...) +{ + va_list args; + DBUG_ENTER("mi_check_print_warning"); + + fflush(stdout); + if (!param->warning_printed && !param->error_printed) + { + if (param->testflag & T_SILENT) + fprintf(stderr,"%s: MyISAM file %s\n",my_progname_short, + param->isam_file_name); + param->out_flag|= O_DATA_LOST; + } + param->warning_printed=1; + va_start(args,fmt); + fprintf(stderr,"%s: warning: ",my_progname_short); + (void) vfprintf(stderr, fmt, args); + (void) fputc('\n',stderr); + fflush(stderr); + va_end(args); + DBUG_VOID_RETURN; +} + +/* VARARGS */ + +void mi_check_print_error(HA_CHECK *param, const char *fmt,...) +{ + va_list args; + DBUG_ENTER("mi_check_print_error"); + DBUG_PRINT("enter",("format: %s",fmt)); + + fflush(stdout); + if (!param->warning_printed && !param->error_printed) + { + if (param->testflag & T_SILENT) + fprintf(stderr,"%s: MyISAM file %s\n",my_progname_short,param->isam_file_name); + param->out_flag|= O_DATA_LOST; + } + param->error_printed|=1; + va_start(args,fmt); + fprintf(stderr,"%s: error: ",my_progname_short); + (void) vfprintf(stderr, fmt, args); + (void) fputc('\n',stderr); + fflush(stderr); + va_end(args); + DBUG_VOID_RETURN; +} + +#include "mi_extrafunc.h" diff --git a/storage/myisam/myisamdef.h b/storage/myisam/myisamdef.h new file mode 100644 index 00000000..f84ad6fa --- /dev/null +++ b/storage/myisam/myisamdef.h @@ -0,0 +1,803 @@ +/* + Copyright (c) 2000, 2012, Oracle and/or its affiliates. + Copyright (c) 2017, 2022, MariaDB Corporation. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* This file is included by all internal myisam files */ + +#include <my_global.h> +#include <myisam.h> /* Structs & some defines */ +#include <myisampack.h> /* packing of keys */ +#include <my_tree.h> +#include <my_pthread.h> +#include <thr_lock.h> +#include <mysql/psi/mysql_file.h> + +C_MODE_START + +typedef struct st_mi_status_info +{ + ha_rows records; /* Rows in table */ + ha_rows del; /* Removed rows */ + my_off_t empty; /* lost space in datafile */ + my_off_t key_empty; /* lost space in indexfile */ + my_off_t key_file_length; + my_off_t data_file_length; + ha_checksum checksum; + my_bool uncacheable; /* Active concurrent insert */ +} MI_STATUS_INFO; + +typedef struct st_mi_state_info +{ + struct + { /* Fileheader */ + uchar file_version[4]; + uchar options[2]; + uchar header_length[2]; + uchar state_info_length[2]; + uchar base_info_length[2]; + uchar base_pos[2]; + uchar key_parts[2]; /* Key parts */ + uchar unique_key_parts[2]; /* Key parts + unique parts */ + uchar keys; /* number of keys in file */ + uchar uniques; /* number of UNIQUE definitions */ + uchar language; /* Language for indexes */ + uchar max_block_size_index; /* max keyblock size */ + uchar fulltext_keys; + uchar not_used; /* To align to 8 */ + } header; + + MI_STATUS_INFO state; + ha_rows split; /* number of split blocks */ + my_off_t dellink; /* Link to next removed block */ + ulonglong auto_increment; + ulong process; /* process that updated table last */ + ulong unique; /* Unique number for this process */ + ulong update_count; /* Updated for each write lock */ + ulong status; + ulong *rec_per_key_part; + ha_checksum checksum; /* Table checksum */ + my_off_t *key_root; /* Start of key trees */ + my_off_t *key_del; /* delete links for trees */ + my_off_t rec_per_key_rows; /* Rows when calculating rec_per_key */ + + ulong sec_index_changed; /* Updated when new sec_index */ + ulong sec_index_used; /* which extra index are in use */ + ulonglong key_map; /* Which keys are in use */ + ulong version; /* timestamp of create */ + time_t create_time; /* Time when created database */ + time_t recover_time; /* Time for last recover */ + time_t check_time; /* Time for last check */ + uint sortkey; /* sorted by this key (not used) */ + uint open_count; + uint8 changed; /* Changed since myisamchk */ + + uint8 dupp_key; /* Lastly processed index with */ + /* violated uniqueness constraint */ + + /* the following isn't saved on disk */ + uint state_diff_length; /* Should be 0 */ + uint state_length; /* Length of state header in file */ + ulong *key_info; +} MI_STATE_INFO; + +#define MI_STATE_INFO_SIZE (24+14*8+7*4+2*2+8) +#define MI_STATE_KEY_SIZE 8U +#define MI_STATE_KEYBLOCK_SIZE 8U +#define MI_STATE_KEYSEG_SIZE 4U +#define MI_STATE_EXTRA_SIZE ((MI_MAX_KEY+MI_MAX_KEY_BLOCK_SIZE)*MI_STATE_KEY_SIZE + MI_MAX_KEY*HA_MAX_KEY_SEG*MI_STATE_KEYSEG_SIZE) +#define MI_KEYDEF_SIZE (2+ 5*2) +#define MI_UNIQUEDEF_SIZE (2+1+1) +#define HA_KEYSEG_SIZE (6+ 2*2 + 4*2) +#define MI_COLUMNDEF_SIZE (2*3+1) +#define MI_BASE_INFO_SIZE (5*8 + 8*4 + 4 + 4*2 + 16) +#define MI_INDEX_BLOCK_MARGIN 16U /* Safety margin for .MYI tables */ + +typedef struct st_mi_base_info +{ + my_off_t keystart; /* Start of keys */ + my_off_t max_data_file_length; + my_off_t max_key_file_length; + my_off_t margin_key_file_length; + ha_rows records, reloc; /* Create information */ + ulong mean_row_length; /* Create information */ + ulong reclength; /* length of unpacked record */ + ulong pack_reclength; /* Length of full packed rec. */ + ulong min_pack_length; + ulong max_pack_length; /* Max possibly length of packed rec.*/ + ulong min_block_length; + ulong fields, /* fields in table */ + pack_fields; /* packed fields in table */ + uint rec_reflength; /* = 2-8 */ + uint key_reflength; /* = 2-8 */ + uint keys; /* same as in state.header */ + uint auto_key; /* Which key-1 is a auto key */ + uint blobs; /* Number of blobs */ + uint pack_bits; /* Length of packed bits */ + uint max_key_block_length; /* Max block length */ + uint max_key_length; /* Max key length */ + /* Extra allocation when using dynamic record format */ + uint extra_alloc_bytes; + uint extra_alloc_procent; + /* The following are from the header */ + uint key_parts, all_key_parts, base_key_parts; +} MI_BASE_INFO; + + + /* Structs used intern in database */ + +typedef struct st_mi_blob /* Info of record */ +{ + ulong offset; /* Offset to blob in record */ + uint pack_length; /* Type of packed length */ + ulong length; /* Calc:ed for each record */ +} MI_BLOB; + + +typedef struct st_mi_isam_pack +{ + ulong header_length; + uint ref_length; + uchar version; +} MI_PACK; + +#define MAX_NONMAPPED_INSERTS 1000 + +typedef struct st_mi_isam_share +{ /* Shared between opens */ + MI_STATE_INFO state; + MI_BASE_INFO base; + MI_KEYDEF ft2_keyinfo; /* Second-level ft-key definition */ + MI_KEYDEF *keyinfo; /* Key definitions */ + MI_UNIQUEDEF *uniqueinfo; /* unique definitions */ + HA_KEYSEG *keyparts; /* key part info */ + MI_COLUMNDEF *rec; /* Pointer to field information */ + MI_PACK pack; /* Data about packed records */ + MI_BLOB *blobs; /* Pointer to blobs */ + LIST *in_use; /* List of threads using this table */ + char *unique_file_name; /* realpath() of index file */ + char *data_file_name, /* Resolved path names from symlinks */ + *index_file_name; + uchar *file_map; /* mem-map of file if possible */ + KEY_CACHE *key_cache; /* ref to the current key cache */ + /* To mark the key cache partitions containing dirty pages for this file */ + ulonglong dirty_part_map; + MI_DECODE_TREE *decode_trees; + uint16 *decode_tables; + /* Function to use for a row checksum. */ + int(*read_record) (struct st_myisam_info *, my_off_t, uchar*); + int(*write_record) (struct st_myisam_info *, const uchar*); + int(*update_record) (struct st_myisam_info *, my_off_t, const uchar*); + int(*delete_record) (struct st_myisam_info *); + int(*read_rnd) (struct st_myisam_info *, uchar*, my_off_t, my_bool); + int(*compare_record) (struct st_myisam_info *, const uchar*); + ha_checksum(*calc_checksum) (struct st_myisam_info *, const uchar*); + /* calculate checksum for a row during check table */ + ha_checksum(*calc_check_checksum)(struct st_myisam_info *, const uchar *); + int(*compare_unique) (struct st_myisam_info *, MI_UNIQUEDEF *, + const uchar *record, my_off_t pos); + size_t (*file_read) (MI_INFO *, uchar *, size_t, my_off_t, myf); + size_t (*file_write) (MI_INFO *, const uchar *, size_t, my_off_t, myf); + invalidator_by_filename invalidator; /* query cache invalidator */ + /* query cache invalidator for changing state */ + invalidator_by_filename chst_invalidator; + ulong this_process; /* processid */ + ulong last_process; /* For table-change-check */ + ulong last_version; /* Version on start */ + ulong options; /* Options used */ + ulong min_pack_length; /* These are used by packed data */ + ulong max_pack_length; + ulong state_diff_length; + uint rec_reflength; /* rec_reflength in use now */ + ulong vreclength; /* full reclength, including vcols */ + uint unique_name_length; + uint32 ftkeys; /* Number of full-text keys + 1 */ + File kfile; /* Shared keyfile */ + File data_file; /* Shared data file */ + int mode; /* mode of file on open */ + uint reopen; /* How many times reopened */ + uint w_locks,r_locks,tot_locks; /* Number of read/write locks */ + uint blocksize; /* blocksize of keyfile */ + myf write_flag; + enum data_file_type data_file_type; + /* Below flag is needed to make log tables work with concurrent insert */ + my_bool is_log_table; + /* This is 1 if they table checksum is of old type */ + my_bool has_null_fields; + my_bool has_varchar_fields; + + my_bool changed, /* If changed since lock */ + global_changed, /* If changed since open */ + not_flushed, temporary, delay_key_write, concurrent_insert; + my_bool deleting; /* we are going to delete this table */ + THR_LOCK lock; + mysql_mutex_t intern_lock; /* Locking for use with _locking */ + mysql_rwlock_t *key_root_lock; + size_t mmaped_length; + uint nonmmaped_inserts; /* counter of writing in non-mmaped + area */ + mysql_rwlock_t mmap_lock; +} MYISAM_SHARE; + + +struct st_myisam_info +{ + MYISAM_SHARE *s; /* Shared between opens */ + MI_STATUS_INFO *state, save_state; + MI_BLOB *blobs; /* Pointer to blobs */ + MI_BIT_BUFF bit_buff; + /* accumulate indexfile changes between writes */ + TREE *bulk_insert; + DYNAMIC_ARRAY *ft1_to_ft2; /* used only in ft1->ft2 conversion */ + MEM_ROOT ft_memroot; /* used by the parser */ + MYSQL_FTPARSER_PARAM *ftparser_param; /* share info between init/deinit */ + void *external_ref; /* For MariaDB TABLE */ + LIST in_use; /* Thread using this table */ + char *filename; /* parameter to open filename */ + uchar *buff, /* Temp area for key */ + *lastkey,*lastkey2; /* Last used search key */ + uchar *first_mbr_key; /* Searhed spatial key */ + uchar *rec_buff; /* Tempbuff for recordpack */ + uchar *int_keypos, /* Save position for next/previous */ + *int_maxpos; /* -""- */ + uint int_nod_flag; /* -""- */ + uint32 int_keytree_version; /* -""- */ + int (*read_record)(struct st_myisam_info*, my_off_t, uchar*); + invalidator_by_filename invalidator; /* query cache invalidator */ + ulong this_unique; /* unique filenumber or thread */ + ulong last_unique; /* last unique number */ + ulong this_loop; /* counter for this open */ + ulong last_loop; /* last used counter */ + my_off_t lastpos, /* Last record position */ + nextpos; /* Position to next record */ + my_off_t save_lastpos; + my_off_t pos; /* Intern variable */ + my_off_t last_keypage; /* Last key page read */ + my_off_t last_search_keypage; /* Last keypage when searching */ + my_off_t dupp_key_pos; + ha_checksum checksum; /* Temp storage for row checksum */ + /* + QQ: the following two xxx_length fields should be removed, + as they are not compatible with parallel repair + */ + ulong packed_length, blob_length; /* Length of found, packed record */ + int dfile; /* The datafile */ + uint open_flag; /* Parameters for open */ + uint opt_flag; /* Optim. for space/speed */ + uint once_flags; /* For MYISAMMRG */ + uint update; /* If file changed since open */ + int lastinx; /* Last used index */ + uint lastkey_length; /* Length of key in lastkey */ + uint last_rkey_length; /* Last length in mi_rkey() */ + enum ha_rkey_function last_key_func; /* CONTAIN, OVERLAP, etc */ + uint save_lastkey_length; + uint pack_key_length; /* For MYISAMMRG */ + uint16 last_used_keyseg; /* For MyISAMMRG */ + int errkey; /* Got last error on this key */ + int lock_type; /* How database was locked */ + int tmp_lock_type; /* When locked by readinfo */ + uint data_changed; /* Somebody has changed data */ + uint save_update; /* When using KEY_READ */ + int save_lastinx; + LIST open_list; + IO_CACHE rec_cache; /* When cacheing records */ + uint preload_buff_size; /* When preloading indexes */ + myf lock_wait; /* is 0 or MY_SHORT_WAIT */ + my_bool was_locked; /* Was locked in panic */ + my_bool intern_lock_locked; /* locked in mi_extra() */ + my_bool append_insert_at_end; /* Set if concurrent insert */ + my_bool quick_mode; + /* If info->buff can't be used for rnext */ + my_bool page_changed; + /* If info->buff has to be reread for rnext */ + my_bool buff_used; + my_bool create_unique_index_by_sort; + index_cond_func_t index_cond_func; /* Index condition function */ + void *index_cond_func_arg; /* parameter for the func */ + rowid_filter_func_t rowid_filter_func; /* rowid filter check function */ + rowid_filter_is_active_func_t rowid_filter_is_active_func; /* is activefunction */ + void *rowid_filter_func_arg; /* parameter for the func */ + THR_LOCK_DATA lock; + uchar *rtree_recursion_state; /* For RTREE */ + int rtree_recursion_depth; +}; + +#define USE_WHOLE_KEY (HA_MAX_KEY_BUFF*2) /* Use whole key in _mi_search() */ +#define F_EXTRA_LCK -1 +/* bits in opt_flag */ +#define MEMMAP_USED 32U +#define REMEMBER_OLD_POS 64U + +#define WRITEINFO_UPDATE_KEYFILE 1U +#define WRITEINFO_NO_UNLOCK 2U + +/* once_flags */ +#define USE_PACKED_KEYS 1U +#define RRND_PRESERVE_LASTINX 2U + +/* bits in state.changed */ +#define STATE_CHANGED 1U +#define STATE_CRASHED 2U +#define STATE_CRASHED_ON_REPAIR 4U +#define STATE_NOT_ANALYZED 8U +#define STATE_NOT_OPTIMIZED_KEYS 16U +#define STATE_NOT_SORTED_PAGES 32U + +/* options to mi_read_cache */ +#define READING_NEXT 1U +#define READING_HEADER 2U + +#define mi_getint(x) ((uint) mi_uint2korr(x) & 32767) +#define mi_putint(x,y,nod) { uint16 boh=(nod ? (uint16) 32768 : 0) + (uint16) (y);\ + mi_int2store(x,boh); } +#define mi_test_if_nod(x) (x[0] & 128 ? info->s->base.key_reflength : 0) +#define mi_report_crashed(A, B) _mi_report_crashed((A), (B), __FILE__, __LINE__) +#define mi_mark_crashed(x) do{(x)->s->state.changed|= STATE_CRASHED; \ + DBUG_PRINT("error", ("Marked table crashed")); \ + mi_report_crashed((x), 0); \ + }while(0) +#define mi_mark_crashed_on_repair(x) do{(x)->s->state.changed|= \ + STATE_CRASHED|STATE_CRASHED_ON_REPAIR; \ + (x)->update|= HA_STATE_CHANGED; \ + DBUG_PRINT("error", \ + ("Marked table crashed")); \ + }while(0) +#define mi_is_crashed(x) ((x)->s->state.changed & STATE_CRASHED) +#define mi_is_crashed_on_repair(x) ((x)->s->state.changed & STATE_CRASHED_ON_REPAIR) +#define mi_print_error(SHARE, ERRNO) \ + mi_report_error((ERRNO), (SHARE)->index_file_name) + +/* Functions to store length of space packed keys, VARCHAR or BLOB keys */ + +#define store_key_length(key,length) \ +{ if ((length) < 255) \ + { *(key)=(length); } \ + else \ + { *(key)=255; mi_int2store((key)+1,(length)); } \ +} + +#define get_key_full_length(length,key) \ +{ if ((uchar) *(key) != 255) \ + length= ((uint) (uchar) *((key)++))+1; \ + else \ + { length=mi_uint2korr((key)+1)+3; (key)+=3; } \ +} + +#define get_key_full_length_rdonly(length,key) \ +{ if ((uchar) *(key) != 255) \ + length= ((uint) (uchar) *((key)))+1; \ + else \ + { length=mi_uint2korr((key)+1)+3; } \ +} + +#define get_pack_length(length) ((length) >= 255 ? 3 : 1) + +#define MI_MIN_BLOCK_LENGTH 20 /* Because of delete-link */ +#define MI_EXTEND_BLOCK_LENGTH 20 /* Don't use to small record-blocks */ +#define MI_SPLIT_LENGTH ((MI_EXTEND_BLOCK_LENGTH+4)*2) +#define MI_MAX_DYN_BLOCK_HEADER 20 /* Max prefix of record-block */ +#define MI_BLOCK_INFO_HEADER_LENGTH 20 +#define MI_DYN_DELETE_BLOCK_HEADER 20 /* length of delete-block-header */ +#define MI_DYN_MAX_BLOCK_LENGTH ((1UL << 24)-4UL) +#define MI_DYN_MAX_ROW_LENGTH (MI_DYN_MAX_BLOCK_LENGTH - MI_SPLIT_LENGTH) +#define MI_DYN_ALIGN_SIZE 4U /* Align blocks on this */ +#define MI_MAX_DYN_HEADER_BYTE 13 /* max header byte for dynamic rows */ +#define MI_MAX_BLOCK_LENGTH (((1U << 24)-1) & (~(MI_DYN_ALIGN_SIZE-1))) +#define MI_REC_BUFF_OFFSET ALIGN_SIZE(MI_DYN_DELETE_BLOCK_HEADER+sizeof(uint32)) + + +#define PACK_TYPE_SELECTED 1U /* Bits in field->pack_type */ +#define PACK_TYPE_SPACE_FIELDS 2U +#define PACK_TYPE_ZERO_FILL 4U +#define MI_FOUND_WRONG_KEY 0x7FFFFFFF /* Impossible value from ha_key_cmp */ + +#define MI_MAX_KEY_BLOCK_SIZE (MI_MAX_KEY_BLOCK_LENGTH/MI_MIN_KEY_BLOCK_LENGTH) +#define MI_BLOCK_SIZE(key_length,data_pointer,key_pointer,block_size) (((((key_length)+(data_pointer)+(key_pointer))*4+(key_pointer)+2)/(block_size)+1)*(block_size)) +#define MI_MAX_KEYPTR_SIZE 5 /* For calculating block lengths */ +#define MI_MIN_KEYBLOCK_LENGTH 50 /* When to split delete blocks */ + +#define MI_MIN_SIZE_BULK_INSERT_TREE 16384U /* this is per key */ +#define MI_MIN_ROWS_TO_USE_BULK_INSERT 100 +#define MI_MIN_ROWS_TO_DISABLE_INDEXES 100 +#define MI_MIN_ROWS_TO_USE_WRITE_CACHE 10 + +/* The UNIQUE check is done with a hashed long key */ + +#define MI_UNIQUE_HASH_TYPE HA_KEYTYPE_ULONG_INT +#define mi_unique_store(A,B) mi_int4store((A),(B)) + +extern mysql_mutex_t THR_LOCK_myisam; +#ifdef DONT_USE_RW_LOCKS +#define mysql_rwlock_wrlock(A) {} +#define mysql_rwlock_rdlock(A) {} +#define mysql_rwlock_unlock(A) {} +#endif + +/* Some extern variables */ + +extern LIST *myisam_open_list; +extern uchar myisam_file_magic[], myisam_pack_file_magic[]; +extern uint myisam_read_vec[], myisam_readnext_vec[]; +extern uint myisam_quick_table_bits; +extern File myisam_log_file; +extern ulong myisam_pid; +extern my_bool (*mi_killed)(MI_INFO *); +extern void _mi_report_crashed(MI_INFO *file, const char *message, + const char *sfile, uint sline); +/* This is used by _mi_calc_xxx_key_length och _mi_store_key */ + +typedef struct st_mi_s_param +{ + uint ref_length, key_length, + n_ref_length, + n_length, totlength, part_of_prev_key, prev_length, pack_marker; + uchar *key, *prev_key, *next_key_pos; + my_bool store_not_null; +} MI_KEY_PARAM; + +/* Prototypes for intern functions */ + +extern int _mi_read_dynamic_record(MI_INFO *info, my_off_t filepos, uchar *buf); +extern int _mi_write_dynamic_record(MI_INFO *, const uchar *); +extern int _mi_update_dynamic_record(MI_INFO *, my_off_t, const uchar *); +extern int _mi_delete_dynamic_record(MI_INFO *info); +extern int _mi_cmp_dynamic_record(MI_INFO *info, const uchar *record); +extern int _mi_read_rnd_dynamic_record(MI_INFO *, uchar *, my_off_t, my_bool); +extern int _mi_write_blob_record(MI_INFO *, const uchar *); +extern int _mi_update_blob_record(MI_INFO *, my_off_t, const uchar *); +extern int _mi_read_static_record(MI_INFO *info, my_off_t filepos, uchar *buf); +extern int _mi_write_static_record(MI_INFO *, const uchar *); +extern int _mi_update_static_record(MI_INFO *, my_off_t, const uchar *); +extern int _mi_delete_static_record(MI_INFO *info); +extern int _mi_cmp_static_record(MI_INFO *info, const uchar *record); +extern int _mi_read_rnd_static_record(MI_INFO *, uchar *, my_off_t, my_bool); +extern int _mi_ck_write(MI_INFO *info, uint keynr, uchar *key, uint length); +extern int _mi_ck_real_write_btree(MI_INFO *info, MI_KEYDEF *keyinfo, + uchar *key, uint key_length, + my_off_t *root, uint comp_flag); +extern int _mi_enlarge_root(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *key, + my_off_t *root); +extern int _mi_insert(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *key, + uchar *anc_buff, uchar *key_pos, uchar *key_buff, + uchar *father_buff, uchar *father_keypos, + my_off_t father_page, my_bool insert_last); +extern int _mi_split_page(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *key, + uchar *buff, uchar *key_buff, my_bool insert_last); +extern uchar *_mi_find_half_pos(uint nod_flag, MI_KEYDEF *keyinfo, + uchar *page, uchar *key, + uint *return_key_length, uchar ** after_key); +extern int _mi_calc_static_key_length(MI_KEYDEF *keyinfo, uint nod_flag, + uchar *key_pos, uchar *org_key, + uchar *key_buff, uchar *key, + MI_KEY_PARAM *s_temp); +extern int _mi_calc_var_key_length(MI_KEYDEF *keyinfo, uint nod_flag, + uchar *key_pos, uchar *org_key, + uchar *key_buff, uchar *key, + MI_KEY_PARAM *s_temp); +extern int _mi_calc_var_pack_key_length(MI_KEYDEF *keyinfo, uint nod_flag, + uchar *key_pos, uchar *org_key, + uchar *prev_key, uchar *key, + MI_KEY_PARAM *s_temp); +extern int _mi_calc_bin_pack_key_length(MI_KEYDEF *keyinfo, uint nod_flag, + uchar *key_pos, uchar *org_key, + uchar *prev_key, uchar *key, + MI_KEY_PARAM *s_temp); +void _mi_store_static_key(MI_KEYDEF *keyinfo, uchar *key_pos, + MI_KEY_PARAM *s_temp); +void _mi_store_var_pack_key(MI_KEYDEF *keyinfo, uchar *key_pos, + MI_KEY_PARAM *s_temp); +void _mi_store_bin_pack_key(MI_KEYDEF *keyinfo, uchar *key_pos, + MI_KEY_PARAM *s_temp); + +extern int _mi_ck_delete(MI_INFO *info, uint keynr, uchar *key, + uint key_length); +extern int _mi_readinfo(MI_INFO *info, int lock_flag, int check_keybuffer); +extern int _mi_writeinfo(MI_INFO *info, uint options); +extern int _mi_test_if_changed(MI_INFO *info); +extern int _mi_mark_file_changed(MI_INFO *info); +extern int _mi_decrement_open_count(MI_INFO *info); +void _mi_report_crashed_ignore(MI_INFO *file, const char *message, + const char *sfile, uint sline); +extern int _mi_check_index(MI_INFO *info, int inx); +extern int _mi_search(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *key, + uint key_len, uint nextflag, my_off_t pos); +extern int _mi_bin_search(struct st_myisam_info *info, MI_KEYDEF *keyinfo, + uchar *page, uchar *key, uint key_len, + uint comp_flag, uchar **ret_pos, uchar *buff, + my_bool *was_last_key); +extern int _mi_seq_search(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, + uchar *key, uint key_len, uint comp_flag, + uchar ** ret_pos, uchar *buff, + my_bool *was_last_key); +extern int _mi_prefix_search(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, + uchar *key, uint key_len, uint comp_flag, + uchar ** ret_pos, uchar *buff, + my_bool *was_last_key); +extern my_off_t _mi_kpos(uint nod_flag, uchar *after_key); +extern void _mi_kpointer(MI_INFO *info, uchar *buff, my_off_t pos); +extern my_off_t _mi_dpos(MI_INFO *info, uint nod_flag, uchar *after_key); +extern my_off_t _mi_rec_pos(MYISAM_SHARE *info, uchar *ptr); +extern void _mi_dpointer(MI_INFO *info, uchar *buff, my_off_t pos); +extern uint _mi_get_static_key(MI_KEYDEF *keyinfo, uint nod_flag, + uchar **page, uchar *key); +extern uint _mi_get_pack_key(MI_KEYDEF *keyinfo, uint nod_flag, uchar **page, + uchar *key); +extern uint _mi_get_binary_pack_key(MI_KEYDEF *keyinfo, uint nod_flag, + uchar ** page_pos, uchar *key); +extern uchar *_mi_get_last_key(MI_INFO *info, MI_KEYDEF *keyinfo, + uchar *keypos, uchar *lastkey, uchar *endpos, + uint *return_key_length); +extern uchar *_mi_get_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, + uchar *key, uchar *keypos, + uint *return_key_length); +extern uint _mi_keylength(MI_KEYDEF *keyinfo, uchar *key); +extern uint _mi_keylength_part(MI_KEYDEF *keyinfo, uchar *key, HA_KEYSEG *end); +extern uchar *_mi_move_key(MI_KEYDEF *keyinfo, uchar *to, uchar *from); +extern int _mi_search_next(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *key, + uint key_length, uint nextflag, my_off_t pos); +extern int _mi_search_first(MI_INFO *info, MI_KEYDEF *keyinfo, my_off_t pos); +extern int _mi_search_last(MI_INFO *info, MI_KEYDEF *keyinfo, my_off_t pos); +extern uchar *_mi_fetch_keypage(MI_INFO *info, MI_KEYDEF *keyinfo, + my_off_t page, int level, uchar *buff, + int return_buffer); +extern int _mi_write_keypage(MI_INFO *info, MI_KEYDEF *keyinfo, my_off_t page, + int level, uchar *buff); +extern int _mi_dispose(MI_INFO *info, MI_KEYDEF *keyinfo, my_off_t pos, + int level); +extern my_off_t _mi_new(MI_INFO *info, MI_KEYDEF *keyinfo, int level); +extern uint _mi_make_key(MI_INFO *info, uint keynr, uchar *key, + const uchar *record, my_off_t filepos); +extern uint _mi_pack_key(MI_INFO *info, uint keynr, uchar *key, + uchar *old, key_part_map keypart_map, + HA_KEYSEG ** last_used_keyseg); +extern int _mi_read_key_record(MI_INFO *info, my_off_t filepos, uchar *buf); +extern int _mi_read_cache(IO_CACHE *info, uchar *buff, my_off_t pos, + size_t length, int re_read_if_possibly); +extern ulonglong retrieve_auto_increment(MI_INFO *info, const uchar *record); + +extern uchar *mi_alloc_rec_buff(MI_INFO *, ulong, uchar **); +#define mi_get_rec_buff_ptr(info,buf) \ + ((((info)->s->options & HA_OPTION_PACK_RECORD) && (buf)) ? \ + (buf) - MI_REC_BUFF_OFFSET : (buf)) +#define mi_get_rec_buff_len(info,buf) \ + (*((uint32 *)(mi_get_rec_buff_ptr(info,buf)))) + +extern size_t _mi_rec_unpack(MI_INFO *info, uchar *to, uchar *from, + ulong reclength); +extern my_bool _mi_rec_check(MI_INFO *info,const uchar *record, uchar *packpos, + ulong packed_length, my_bool with_checkum); +extern int _mi_write_part_record(MI_INFO *info, my_off_t filepos, ulong length, + my_off_t next_filepos, uchar ** record, + ulong *reclength, int *flag); +extern void _mi_print_key(FILE *stream, HA_KEYSEG *keyseg, const uchar *key, + uint length); +extern my_bool _mi_read_pack_info(MI_INFO *info, pbool fix_keys); +extern int _mi_read_pack_record(MI_INFO *info, my_off_t filepos, uchar *buf); +extern int _mi_read_rnd_pack_record(MI_INFO *, uchar *, my_off_t, my_bool); +extern int _mi_pack_rec_unpack(MI_INFO *info, MI_BIT_BUFF *bit_buff, + uchar *to, uchar *from, ulong reclength); +extern ulonglong mi_safe_mul(ulonglong a, ulonglong b); +extern int _mi_ft_update(MI_INFO *info, uint keynr, uchar *keybuf, + const uchar *oldrec, const uchar *newrec, + my_off_t pos); +extern my_bool mi_yield_and_check_if_killed(MI_INFO *info, int inx); +extern my_bool mi_killed_standalone(MI_INFO *); + +struct st_sort_info; + + +typedef struct st_mi_block_info /* Parameter to _mi_get_block_info */ +{ + uchar header[MI_BLOCK_INFO_HEADER_LENGTH]; + ulong rec_len; + ulong data_len; + ulong block_len; + ulong blob_len; + my_off_t filepos; + my_off_t next_filepos; + my_off_t prev_filepos; + uint second_read; + uint offset; +} MI_BLOCK_INFO; + + +struct st_sort_key_blocks /* Used when sorting */ +{ + uchar *buff, *end_pos; + uchar lastkey[HA_MAX_POSSIBLE_KEY_BUFF]; + uint last_length; + int inited; +}; + + +struct st_sort_ftbuf +{ + uchar *buf, *end; + int count; + uchar lastkey[HA_MAX_KEY_BUFF]; +}; + +/* bits in return from _mi_get_block_info */ + +#define BLOCK_FIRST 1U +#define BLOCK_LAST 2U +#define BLOCK_DELETED 4U +#define BLOCK_ERROR 8U /* Wrong data */ +#define BLOCK_SYNC_ERROR 16U /* Right data at wrong place */ +#define BLOCK_FATAL_ERROR 32U /* hardware-error */ + +#define NEED_MEM ((uint) 10*4*(IO_SIZE+32)+32) /* Nead for recursion */ +#define MAXERR 20 +#define BUFFERS_WHEN_SORTING 16 /* Alloc for sort-key-tree */ +#define WRITE_COUNT MY_HOW_OFTEN_TO_WRITE +#define INDEX_TMP_EXT ".TMM" +#define DATA_TMP_EXT ".TMD" + +#define UPDATE_TIME 1U +#define UPDATE_STAT 2U +#define UPDATE_SORT 4U +#define UPDATE_AUTO_INC 8U +#define UPDATE_OPEN_COUNT 16U + +/* We use MY_ALIGN_DOWN here mainly to ensure that we get stable values for mysqld --help ) */ +#define KEY_BUFFER_INIT MY_ALIGN_DOWN(1024L*1024L-MALLOC_OVERHEAD, IO_SIZE) +#define READ_BUFFER_INIT MY_ALIGN_DOWN(1024L*256L-MALLOC_OVERHEAD, 1024) +#define SORT_BUFFER_INIT MY_ALIGN_DOWN(1024L*1024L*128L-MALLOC_OVERHEAD, 1024) +#define MIN_SORT_BUFFER 4096U + +enum myisam_log_commands +{ + MI_LOG_OPEN, MI_LOG_WRITE, MI_LOG_UPDATE, MI_LOG_DELETE, MI_LOG_CLOSE, + MI_LOG_EXTRA, MI_LOG_LOCK, MI_LOG_DELETE_ALL +}; + +#define myisam_log(a,b,c,d) if (myisam_log_file >= 0) _myisam_log(a,b,c,d) +#define myisam_log_command(a,b,c,d,e) if (myisam_log_file >= 0) _myisam_log_command(a,b,c,d,e) +#define myisam_log_record(a,b,c,d,e) if (myisam_log_file >= 0) _myisam_log_record(a,b,c,d,e) + +#define fast_mi_writeinfo(INFO) if (!(INFO)->s->tot_locks) (void) _mi_writeinfo((INFO),0) +#define fast_mi_readinfo(INFO) ((INFO)->lock_type == F_UNLCK) && _mi_readinfo((INFO),F_RDLCK,1) + +extern uint _mi_get_block_info(MI_BLOCK_INFO *, File, my_off_t); +extern uint _mi_rec_pack(MI_INFO *info, uchar *to, const uchar *from); +extern uint _mi_pack_get_block_info(MI_INFO *myisam, MI_BIT_BUFF *bit_buff, + MI_BLOCK_INFO *info, uchar **rec_buff_p, + File file, my_off_t filepos); +extern void _mi_store_blob_length(uchar *pos, uint pack_length, uint length); +extern void _myisam_log(enum myisam_log_commands command, MI_INFO *info, + const uchar *buffert, uint length); +extern void _myisam_log_command(enum myisam_log_commands command, + MI_INFO *info, const uchar *buffert, + uint length, int result); +extern void _myisam_log_record(enum myisam_log_commands command, MI_INFO *info, + const uchar *record, my_off_t filepos, + int result); +extern void mi_report_error(int errcode, const char *file_name); +extern my_bool _mi_memmap_file(MI_INFO *info); +extern void _mi_unmap_file(MI_INFO *info); +extern uint save_pack_length(uint version, uchar *block_buff, ulong length); +extern uint calc_pack_length(uint version, ulong length); +extern size_t mi_mmap_pread(MI_INFO *info, uchar *Buffer, + size_t Count, my_off_t offset, myf MyFlags); +extern size_t mi_mmap_pwrite(MI_INFO *info, const uchar *Buffer, + size_t Count, my_off_t offset, myf MyFlags); +extern size_t mi_nommap_pread(MI_INFO *info, uchar *Buffer, + size_t Count, my_off_t offset, myf MyFlags); +extern size_t mi_nommap_pwrite(MI_INFO *info, const uchar *Buffer, + size_t Count, my_off_t offset, myf MyFlags); + +uint mi_state_info_write(File file, MI_STATE_INFO *state, uint pWrite); +uchar *mi_state_info_read(uchar *ptr, MI_STATE_INFO *state); +uint mi_state_info_read_dsk(File file, MI_STATE_INFO *state, my_bool pRead); +uint mi_base_info_write(File file, MI_BASE_INFO *base); +uchar *mi_n_base_info_read(uchar *ptr, MI_BASE_INFO *base); +int mi_keyseg_write(File file, const HA_KEYSEG *keyseg); +uchar *mi_keyseg_read(uchar *ptr, HA_KEYSEG *keyseg); +uint mi_keydef_write(File file, MI_KEYDEF *keydef); +uchar *mi_keydef_read(uchar *ptr, MI_KEYDEF *keydef); +uint mi_uniquedef_write(File file, MI_UNIQUEDEF *keydef); +uchar *mi_uniquedef_read(uchar *ptr, MI_UNIQUEDEF *keydef); +uint mi_recinfo_write(File file, MI_COLUMNDEF *recinfo); +uchar *mi_recinfo_read(uchar *ptr, MI_COLUMNDEF *recinfo); +extern int mi_disable_indexes(MI_INFO *info); +extern int mi_enable_indexes(MI_INFO *info); +extern int mi_indexes_are_disabled(MI_INFO *info); +ulong _mi_calc_total_blob_length(MI_INFO *info, const uchar *record); +ha_checksum mi_checksum(MI_INFO *info, const uchar *buf); +ha_checksum mi_static_checksum(MI_INFO *info, const uchar *buf); +my_bool mi_check_unique(MI_INFO *info, MI_UNIQUEDEF *def, const uchar *record, + ha_checksum unique_hash, my_off_t pos); +ha_checksum mi_unique_hash(MI_UNIQUEDEF *def, const uchar *buf); +int _mi_cmp_static_unique(MI_INFO *info, MI_UNIQUEDEF *def, + const uchar *record, my_off_t pos); +int _mi_cmp_dynamic_unique(MI_INFO *info, MI_UNIQUEDEF *def, + const uchar *record, my_off_t pos); +int mi_unique_comp(MI_UNIQUEDEF *def, const uchar *a, const uchar *b, + my_bool null_are_equal); +my_bool mi_get_status(void *param, my_bool concurrent_insert); +void mi_update_status(void *param); +void mi_restore_status(void *param); +void mi_copy_status(void *to, void *from); +my_bool mi_check_status(void *param); +void mi_fix_status(MI_INFO *org_table, MI_INFO *new_table); +extern MI_INFO *test_if_reopen(char *filename); +my_bool check_table_is_closed(const char *name, const char *where); +int mi_open_datafile(MI_INFO *info, MYISAM_SHARE *share); + +int mi_open_keyfile(MYISAM_SHARE *share); +void mi_setup_functions(MYISAM_SHARE *share); +my_bool mi_dynmap_file(MI_INFO *info, my_off_t size); +int mi_munmap_file(MI_INFO *info); +void mi_remap_file(MI_INFO *info, my_off_t size); + +check_result_t mi_check_index_tuple(MI_INFO *info, uint keynr, uchar *record); + + /* Functions needed by mi_check */ +int killed_ptr(HA_CHECK *param); +void mi_check_print_error(HA_CHECK *param, const char *fmt, ...); +void mi_check_print_warning(HA_CHECK *param, const char *fmt, ...); +void mi_check_print_info(HA_CHECK *param, const char *fmt, ...); +pthread_handler_t thr_find_all_keys(void *arg); +extern void mi_set_index_cond_func(MI_INFO *info, index_cond_func_t check_func, + void *func_arg); +extern void mi_set_rowid_filter_func(MI_INFO *info, + rowid_filter_func_t check_func, + rowid_filter_is_active_func_t is_active_func, + void *func_arg); +int flush_blocks(HA_CHECK *param, KEY_CACHE *key_cache, File file, + ulonglong *dirty_part_map); + +#ifdef HAVE_PSI_INTERFACE +extern PSI_mutex_key mi_key_mutex_MYISAM_SHARE_intern_lock, + mi_key_mutex_MI_SORT_INFO_mutex, mi_key_mutex_MI_CHECK_print_msg; + +extern PSI_rwlock_key mi_key_rwlock_MYISAM_SHARE_key_root_lock, + mi_key_rwlock_MYISAM_SHARE_mmap_lock; + +extern PSI_cond_key mi_key_cond_MI_SORT_INFO_cond; + +extern PSI_file_key mi_key_file_datatmp, mi_key_file_dfile, mi_key_file_kfile, + mi_key_file_log; + +extern PSI_thread_key mi_key_thread_find_all_keys; + +void init_myisam_psi_keys(); +#else +#define init_myisam_psi_keys() do { } while(0) +#endif /* HAVE_PSI_INTERFACE */ + +extern PSI_memory_key mi_key_memory_MYISAM_SHARE; +extern PSI_memory_key mi_key_memory_MI_INFO; +extern PSI_memory_key mi_key_memory_MI_INFO_ft1_to_ft2; +extern PSI_memory_key mi_key_memory_MI_INFO_bulk_insert; +extern PSI_memory_key mi_key_memory_record_buffer; +extern PSI_memory_key mi_key_memory_FTB; +extern PSI_memory_key mi_key_memory_FT_INFO; +extern PSI_memory_key mi_key_memory_FTPARSER_PARAM; +extern PSI_memory_key mi_key_memory_ft_memroot; +extern PSI_memory_key mi_key_memory_ft_stopwords; +extern PSI_memory_key mi_key_memory_MI_SORT_PARAM; +extern PSI_memory_key mi_key_memory_MI_SORT_PARAM_wordroot; +extern PSI_memory_key mi_key_memory_SORT_FT_BUF; +extern PSI_memory_key mi_key_memory_SORT_KEY_BLOCKS; +extern PSI_memory_key mi_key_memory_filecopy; +extern PSI_memory_key mi_key_memory_SORT_INFO_buffer; +extern PSI_memory_key mi_key_memory_MI_DECODE_TREE; +extern PSI_memory_key mi_key_memory_MYISAM_SHARE_decode_tables; +extern PSI_memory_key mi_key_memory_preload_buffer; +extern PSI_memory_key mi_key_memory_stPageList_pages; +extern PSI_memory_key mi_key_memory_keycache_thread_var; + +C_MODE_END diff --git a/storage/myisam/myisamlog.c b/storage/myisam/myisamlog.c new file mode 100644 index 00000000..40d473dc --- /dev/null +++ b/storage/myisam/myisamlog.c @@ -0,0 +1,850 @@ +/* + Copyright (c) 2000, 2010, Oracle and/or its affiliates + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* write whats in isam.log */ + +#ifndef USE_MY_FUNC +#define USE_MY_FUNC +#endif + +#include "myisamdef.h" +#include <my_tree.h> +#include <stdarg.h> +#ifdef HAVE_GETRUSAGE +#include <sys/resource.h> +#endif + +#define FILENAME(A) (A ? A->show_name : "Unknown") + +struct file_info { + long process; + int filenr,id; + uint rnd; + char *name, *show_name; + uchar *record; + MI_INFO *isam; + my_bool closed, used; + ulong accessed; +}; + +struct test_if_open_param { + char * name; + int max_id; +}; + +struct st_access_param +{ + ulong min_accessed; + struct file_info *found; +}; + +#define NO_FILEPOS (ulong) ~0L + +extern int main(int argc,char * *argv); +static void get_options(int *argc,char ***argv); +static int examine_log(char * file_name,char **table_names); +static int read_string(IO_CACHE *file,uchar* *to,uint length); +static int file_info_compare(void *cmp_arg, void *a,void *b); +static int test_if_open(struct file_info *key,element_count count, + struct test_if_open_param *param); +static void fix_blob_pointers(MI_INFO *isam,uchar *record); +static int test_when_accessed(struct file_info *key,element_count count, + struct st_access_param *access_param); +static int file_info_free(void*, TREE_FREE, void *); +static int close_some_file(TREE *tree); +static int reopen_closed_file(TREE *tree,struct file_info *file_info); +static int find_record_with_key(struct file_info *file_info,uchar *record); +static void printf_log(const char *str,...); +static my_bool cmp_filename(struct file_info *file_info,char * name); + +static uint verbose=0,update=0,test_info=0,max_files=0,re_open_count=0, + recover=0,prefix_remove=0,opt_processes=0; +static char *log_filename=0, *filepath=0, *write_filename=0; +static char *record_pos_file= 0; +static ulong com_count[10][3],number_of_commands=(ulong) ~0L, + isamlog_process; +static my_off_t isamlog_filepos,start_offset=0,record_pos= HA_OFFSET_ERROR; +static const char *command_name[]= +{"open","write","update","delete","close","extra","lock","re-open", + "delete-all", NullS}; + + +int main(int argc, char **argv) +{ + int error,i,first; + ulong total_count,total_error,total_recover; + MY_INIT(argv[0]); + + log_filename=myisam_log_filename; + get_options(&argc,&argv); + /* Number of MyISAM files we can have open at one time */ + max_files= (my_set_max_open_files(MY_MIN(max_files,8))-6)/2; + if (update) + printf("Trying to %s MyISAM files according to log '%s'\n", + (recover ? "recover" : "update"),log_filename); + error= examine_log(log_filename,argv); + if (update && ! error) + puts("Tables updated successfully"); + total_count=total_error=total_recover=0; + for (i=first=0 ; command_name[i] ; i++) + { + if (com_count[i][0]) + { + if (!first++) + { + if (verbose || update) + puts(""); + puts("Commands Used count Errors Recover errors"); + } + printf("%-12s%9ld%10ld%17ld\n",command_name[i],com_count[i][0], + com_count[i][1],com_count[i][2]); + total_count+=com_count[i][0]; + total_error+=com_count[i][1]; + total_recover+=com_count[i][2]; + } + } + if (total_count) + printf("%-12s%9ld%10ld%17ld\n","Total",total_count,total_error, + total_recover); + if (re_open_count) + printf("Had to do %d re-open because of too few possibly open files\n", + re_open_count); + (void) mi_panic(HA_PANIC_CLOSE); + my_free_open_file_info(); + my_end(test_info ? MY_CHECK_ERROR | MY_GIVE_INFO : MY_CHECK_ERROR); + exit(error); + return 0; /* No compiler warning */ +} /* main */ + + +static void get_options(register int *argc, register char ***argv) +{ + int help,version; + const char *pos,*usage; + char option; + + help=0; + usage="Usage: %s [-?iruvDIV] [-c #] [-f #] [-F filepath/] [-o #] [-R file recordpos] [-w write_file] [log-filename [table ...]] \n"; + pos=""; + + while (--*argc > 0 && *(pos = *(++*argv)) == '-' ) { + while (*++pos) + { + version=0; + switch((option=*pos)) { + case '#': + DBUG_PUSH (++pos); + pos=" "; /* Skip rest of arg */ + break; + case 'c': + if (! *++pos) + { + if (!--*argc) + goto err; + else + pos= *(++*argv); + } + number_of_commands=(ulong) atol(pos); + pos=" "; + break; + case 'u': + update=1; + break; + case 'f': + if (! *++pos) + { + if (!--*argc) + goto err; + else + pos= *(++*argv); + } + max_files=(uint) atoi(pos); + pos=" "; + break; + case 'i': + test_info=1; + break; + case 'o': + if (! *++pos) + { + if (!--*argc) + goto err; + else + pos= *(++*argv); + } + start_offset=(my_off_t) strtoll(pos,NULL,10); + pos=" "; + break; + case 'p': + if (! *++pos) + { + if (!--*argc) + goto err; + else + pos= *(++*argv); + } + prefix_remove=atoi(pos); + break; + case 'r': + update=1; + recover++; + break; + case 'P': + opt_processes=1; + break; + case 'R': + if (! *++pos) + { + if (!--*argc) + goto err; + else + pos= *(++*argv); + } + record_pos_file=(char*) pos; + if (!--*argc) + goto err; + record_pos=(my_off_t) strtoll(*(++*argv),NULL,10); + pos=" "; + break; + case 'v': + verbose++; + break; + case 'w': + if (! *++pos) + { + if (!--*argc) + goto err; + else + pos= *(++*argv); + } + write_filename=(char*) pos; + pos=" "; + break; + case 'F': + if (! *++pos) + { + if (!--*argc) + goto err; + else + pos= *(++*argv); + } + filepath= (char*) pos; + pos=" "; + break; + case 'V': + version=1; + /* Fall through */ + case 'I': + case '?': + printf("%s Ver 1.4 for %s at %s\n",my_progname,SYSTEM_TYPE, + MACHINE_TYPE); + puts("By Monty, for your professional use\n"); + if (version) + break; + puts("Write info about whats in a MyISAM log file."); + printf("If no file name is given %s is used\n",log_filename); + puts(""); + printf(usage,my_progname); + puts(""); + puts("Options: -? or -I \"Info\" -V \"version\" -c \"do only # commands\""); + puts(" -f \"max open files\" -F \"filepath\" -i \"extra info\""); + puts(" -o \"offset\" -p # \"remove # components from path\""); + puts(" -r \"recover\" -R \"file recordposition\""); + puts(" -u \"update\" -v \"verbose\" -w \"write file\""); + puts(" -D \"myisam compiled with DBUG\" -P \"processes\""); + puts("\nOne can give a second and a third '-v' for more verbose."); + puts("Normally one does a update (-u)."); + puts("If a recover is done all writes and all possibly updates and deletes is done\nand errors are only counted."); + puts("If one gives table names as arguments only these tables will be updated\n"); + help=1; + break; + default: + printf("illegal option: \"-%c\"\n",*pos); + break; + } + } + } + if (! *argc) + { + if (help) + exit(0); + (*argv)++; + } + if (*argc >= 1) + { + log_filename=(char*) pos; + (*argc)--; + (*argv)++; + } + return; + err: + (void) fprintf(stderr,"option \"%c\" used without or with wrong argument\n", + option); + exit(1); +} + + +static int examine_log(char * file_name, char **table_names) +{ + uint command,result,files_open; + ulong access_time,length; + my_off_t filepos; + int lock_command,mi_result; + char isam_file_name[FN_REFLEN],llbuff[21],llbuff2[21]; + uchar head[20]; + uchar* buff; + struct test_if_open_param open_param; + IO_CACHE cache; + File file; + FILE *write_file; + enum ha_extra_function extra_command; + TREE tree; + struct file_info file_info,*curr_file_info; + DBUG_ENTER("examine_log"); + + if ((file=my_open(file_name,O_RDONLY,MYF(MY_WME))) < 0) + DBUG_RETURN(1); + write_file=0; + if (write_filename) + { + if (!(write_file=my_fopen(write_filename,O_WRONLY,MYF(MY_WME)))) + { + my_close(file,MYF(0)); + DBUG_RETURN(1); + } + } + + init_io_cache(&cache,file,0,READ_CACHE,start_offset,0,MYF(0)); + bzero((uchar*) com_count,sizeof(com_count)); + init_tree(&tree,0,0,sizeof(file_info),(qsort_cmp2) file_info_compare, + file_info_free, NULL, MYF(MY_TREE_WITH_DELETE)); + (void) init_key_cache(dflt_key_cache,KEY_CACHE_BLOCK_SIZE,KEY_CACHE_SIZE, + 0, 0, 0, 0); + + files_open=0; access_time=0; + while (access_time++ != number_of_commands && + !my_b_read(&cache,(uchar*) head,9)) + { + isamlog_filepos=my_b_tell(&cache)-9L; + file_info.filenr= mi_uint2korr(head+1); + isamlog_process=file_info.process=(long) mi_uint4korr(head+3); + if (!opt_processes) + file_info.process=0; + result= mi_uint2korr(head+7); + if ((curr_file_info=(struct file_info*) tree_search(&tree, &file_info, + tree.custom_arg))) + { + curr_file_info->accessed=access_time; + if (update && curr_file_info->used && curr_file_info->closed) + { + if (reopen_closed_file(&tree,curr_file_info)) + { + command=sizeof(com_count)/sizeof(com_count[0][0])/3; + result=0; + goto com_err; + } + } + } + command=(uint) head[0]; + if (command < sizeof(com_count)/sizeof(com_count[0][0])/3 && + (!table_names[0] || (curr_file_info && curr_file_info->used))) + { + com_count[command][0]++; + if (result) + com_count[command][1]++; + } + switch ((enum myisam_log_commands) command) { + case MI_LOG_OPEN: + if (!table_names[0]) + { + com_count[command][0]--; /* Must be counted explicite */ + if (result) + com_count[command][1]--; + } + + if (curr_file_info) + printf("\nWarning: %s is opened with same process and filenumber\n" + "Maybe you should use the -P option ?\n", + curr_file_info->show_name); + if (my_b_read(&cache,(uchar*) head,2)) + goto err; + buff= 0; + file_info.name=0; + file_info.show_name=0; + file_info.record=0; + if (read_string(&cache, &buff, (uint) mi_uint2korr(head))) + goto err; + { + uint i; + char *pos,*to; + + /* Fix if old DOS files to new format */ + for (pos=file_info.name=(char*)buff; (pos=strchr(pos,'\\')) ; pos++) + *pos= '/'; + + pos=file_info.name; + for (i=0 ; i < prefix_remove ; i++) + { + char *next; + if (!(next=strchr(pos,'/'))) + break; + pos=next+1; + } + to=isam_file_name; + if (filepath) + to=convert_dirname(isam_file_name,filepath,NullS); + strmov(to,pos); + fn_ext(isam_file_name)[0]=0; /* Remove extension */ + } + open_param.name=file_info.name; + open_param.max_id=0; + (void) tree_walk(&tree,(tree_walk_action) test_if_open,(void*) &open_param, + left_root_right); + file_info.id=open_param.max_id+1; + /* + * In the line below +10 is added to accommodate '<' and '>' chars + * plus '\0' at the end, so that there is place for 7 digits. + * It is improbable that same table can have that many entries in + * the table cache. + * The additional space is needed for the sprintf commands two lines + * below. + */ + file_info.show_name=my_memdup(PSI_NOT_INSTRUMENTED, isam_file_name, + (uint) strlen(isam_file_name)+10, + MYF(MY_WME)); + if (file_info.id > 1) + sprintf(strend(file_info.show_name),"<%d>",file_info.id); + file_info.closed=1; + file_info.accessed=access_time; + file_info.used=1; + if (table_names[0]) + { + char **name; + file_info.used=0; + for (name=table_names ; *name ; name++) + { + if (!strcmp(*name,isam_file_name)) + file_info.used=1; /* Update/log only this */ + } + } + if (update && file_info.used) + { + if (files_open >= max_files) + { + if (close_some_file(&tree)) + goto com_err; + files_open--; + } + if (!(file_info.isam= mi_open(isam_file_name,O_RDWR, + HA_OPEN_WAIT_IF_LOCKED))) + goto com_err; + if (!(file_info.record=my_malloc(PSI_NOT_INSTRUMENTED, + file_info.isam->s->base.reclength, MYF(MY_WME)))) + goto end; + files_open++; + file_info.closed=0; + } + (void) tree_insert(&tree, (uchar*) &file_info, 0, tree.custom_arg); + if (file_info.used) + { + if (verbose && !record_pos_file) + printf_log("%s: open -> %d",file_info.show_name, file_info.filenr); + com_count[command][0]++; + if (result) + com_count[command][1]++; + } + break; + case MI_LOG_CLOSE: + if (verbose && !record_pos_file && + (!table_names[0] || (curr_file_info && curr_file_info->used))) + printf_log("%s: %s -> %d",FILENAME(curr_file_info), + command_name[command],result); + if (curr_file_info) + { + if (!curr_file_info->closed) + files_open--; + (void) tree_delete(&tree, (uchar*) curr_file_info, 0, tree.custom_arg); + } + break; + case MI_LOG_EXTRA: + if (my_b_read(&cache,(uchar*) head,1)) + goto err; + extra_command=(enum ha_extra_function) head[0]; + if (verbose && !record_pos_file && + (!table_names[0] || (curr_file_info && curr_file_info->used))) + printf_log("%s: %s(%d) -> %d",FILENAME(curr_file_info), + command_name[command], (int) extra_command,result); + if (update && curr_file_info && !curr_file_info->closed) + { + if (mi_extra(curr_file_info->isam, extra_command, 0) != (int) result) + { + fflush(stdout); + (void) fprintf(stderr, + "Warning: error %d, expected %d on command %s at %s\n", + my_errno,result,command_name[command], + llstr(isamlog_filepos,llbuff)); + fflush(stderr); + } + } + break; + case MI_LOG_DELETE: + if (my_b_read(&cache,(uchar*) head,8)) + goto err; + filepos=mi_sizekorr(head); + if (verbose && (!record_pos_file || + ((record_pos == filepos || record_pos == NO_FILEPOS) && + !cmp_filename(curr_file_info,record_pos_file))) && + (!table_names[0] || (curr_file_info && curr_file_info->used))) + printf_log("%s: %s at %ld -> %d",FILENAME(curr_file_info), + command_name[command],(long) filepos,result); + if (update && curr_file_info && !curr_file_info->closed) + { + if (mi_rrnd(curr_file_info->isam,curr_file_info->record,filepos)) + { + if (!recover) + goto com_err; + if (verbose) + printf_log("error: Didn't find row to delete with mi_rrnd"); + com_count[command][2]++; /* Mark error */ + } + mi_result=mi_delete(curr_file_info->isam,curr_file_info->record); + if ((mi_result == 0 && result) || + (mi_result && (uint) my_errno != result)) + { + if (!recover) + goto com_err; + if (mi_result) + com_count[command][2]++; /* Mark error */ + if (verbose) + printf_log("error: Got result %d from mi_delete instead of %d", + mi_result, result); + } + } + break; + case MI_LOG_WRITE: + case MI_LOG_UPDATE: + if (my_b_read(&cache,(uchar*) head,12)) + goto err; + filepos=mi_sizekorr(head); + length=mi_uint4korr(head+8); + buff=0; + if (read_string(&cache,&buff,(uint) length)) + goto err; + if ((!record_pos_file || + ((record_pos == filepos || record_pos == NO_FILEPOS) && + !cmp_filename(curr_file_info,record_pos_file))) && + (!table_names[0] || (curr_file_info && curr_file_info->used))) + { + if (write_file && + (my_fwrite(write_file,buff,length,MYF(MY_WAIT_IF_FULL | MY_NABP)))) + goto end; + if (verbose) + printf_log("%s: %s at %ld, length=%ld -> %d", + FILENAME(curr_file_info), + command_name[command], filepos,length,result); + } + if (update && curr_file_info && !curr_file_info->closed) + { + if (curr_file_info->isam->s->base.blobs) + fix_blob_pointers(curr_file_info->isam,buff); + if ((enum myisam_log_commands) command == MI_LOG_UPDATE) + { + if (mi_rrnd(curr_file_info->isam,curr_file_info->record,filepos)) + { + if (!recover) + { + result=0; + goto com_err; + } + if (verbose) + printf_log("error: Didn't find row to update with mi_rrnd"); + if (recover == 1 || result || + find_record_with_key(curr_file_info,buff)) + { + com_count[command][2]++; /* Mark error */ + break; + } + } + mi_result=mi_update(curr_file_info->isam,curr_file_info->record, + buff); + if ((mi_result == 0 && result) || + (mi_result && (uint) my_errno != result)) + { + if (!recover) + goto com_err; + if (verbose) + printf_log("error: Got result %d from mi_update instead of %d", + mi_result, result); + if (mi_result) + com_count[command][2]++; /* Mark error */ + } + } + else + { + mi_result=mi_write(curr_file_info->isam,buff); + if ((mi_result == 0 && result) || + (mi_result && (uint) my_errno != result)) + { + if (!recover) + goto com_err; + if (verbose) + printf_log("error: Got result %d from mi_write instead of %d", + mi_result, result); + if (mi_result) + com_count[command][2]++; /* Mark error */ + } + if (!recover && filepos != curr_file_info->isam->lastpos) + { + printf("error: Wrote at position: %s, should have been %s", + llstr(curr_file_info->isam->lastpos,llbuff), + llstr(filepos,llbuff2)); + goto end; + } + } + } + my_free(buff); + break; + case MI_LOG_LOCK: + if (my_b_read(&cache,(uchar*) head,sizeof(lock_command))) + goto err; + memcpy(&lock_command, head, sizeof(lock_command)); + if (verbose && !record_pos_file && + (!table_names[0] || (curr_file_info && curr_file_info->used))) + printf_log("%s: %s(%d) -> %d\n",FILENAME(curr_file_info), + command_name[command],lock_command,result); + if (update && curr_file_info && !curr_file_info->closed) + { + if (mi_lock_database(curr_file_info->isam,lock_command) != + (int) result) + goto com_err; + } + break; + case MI_LOG_DELETE_ALL: + if (verbose && !record_pos_file && + (!table_names[0] || (curr_file_info && curr_file_info->used))) + printf_log("%s: %s -> %d\n",FILENAME(curr_file_info), + command_name[command],result); + break; + default: + fflush(stdout); + (void) fprintf(stderr, + "Error: found unknown command %d in logfile, aborted\n", + command); + fflush(stderr); + goto end; + } + } + end_key_cache(dflt_key_cache,1); + delete_tree(&tree, 0); + (void) end_io_cache(&cache); + (void) my_close(file,MYF(0)); + if (write_file && my_fclose(write_file,MYF(MY_WME))) + DBUG_RETURN(1); + DBUG_RETURN(0); + + err: + fflush(stdout); + (void) fprintf(stderr,"Got error %d when reading from logfile\n",my_errno); + fflush(stderr); + goto end; + com_err: + fflush(stdout); + (void) fprintf(stderr,"Got error %d, expected %d on command %s at %s\n", + my_errno,result,command_name[command], + llstr(isamlog_filepos,llbuff)); + fflush(stderr); + end: + end_key_cache(dflt_key_cache, 1); + delete_tree(&tree, 0); + (void) end_io_cache(&cache); + (void) my_close(file,MYF(0)); + if (write_file) + (void) my_fclose(write_file,MYF(MY_WME)); + DBUG_RETURN(1); +} + + +static int read_string(IO_CACHE *file, register uchar* *to, register uint length) +{ + DBUG_ENTER("read_string"); + + if (*to) + my_free(*to); + if (!(*to= (uchar*) my_malloc(PSI_NOT_INSTRUMENTED, length+1,MYF(MY_WME))) || + my_b_read(file,(uchar*) *to,length)) + { + if (*to) + my_free(*to); + *to= 0; + DBUG_RETURN(1); + } + *((uchar*) *to+length)= '\0'; + DBUG_RETURN (0); +} /* read_string */ + + +static int file_info_compare(void* cmp_arg __attribute__((unused)), + void *a, void *b) +{ + long lint; + + if ((lint=((struct file_info*) a)->process - + ((struct file_info*) b)->process)) + return lint < 0L ? -1 : 1; + return ((struct file_info*) a)->filenr - ((struct file_info*) b)->filenr; +} + + /* ARGSUSED */ + +static int test_if_open (struct file_info *key, + element_count count __attribute__((unused)), + struct test_if_open_param *param) +{ + if (!strcmp(key->name,param->name) && key->id > param->max_id) + param->max_id=key->id; + return 0; +} + + +static void fix_blob_pointers(MI_INFO *info, uchar *record) +{ + uchar *pos; + MI_BLOB *blob,*end; + + pos=record+info->s->base.reclength; + for (end=info->blobs+info->s->base.blobs, blob= info->blobs; + blob != end ; + blob++) + { + memcpy(record+blob->offset+blob->pack_length, &pos, sizeof(char*)); + pos+=_mi_calc_blob_length(blob->pack_length,record+blob->offset); + } +} + + /* close the file with hasn't been accessed for the longest time */ + /* ARGSUSED */ + +static int test_when_accessed (struct file_info *key, + element_count count __attribute__((unused)), + struct st_access_param *access_param) +{ + if (key->accessed < access_param->min_accessed && ! key->closed) + { + access_param->min_accessed=key->accessed; + access_param->found=key; + } + return 0; +} + + +static int file_info_free(void* arg, TREE_FREE mode __attribute__((unused)), + void *unused __attribute__((unused))) +{ + struct file_info *fileinfo= arg; + DBUG_ENTER("file_info_free"); + if (update) + { + if (!fileinfo->closed) + (void) mi_close(fileinfo->isam); + if (fileinfo->record) + my_free(fileinfo->record); + } + my_free(fileinfo->name); + my_free(fileinfo->show_name); + DBUG_RETURN(0); +} + + + +static int close_some_file(TREE *tree) +{ + struct st_access_param access_param; + + access_param.min_accessed=LONG_MAX; + access_param.found=0; + + (void) tree_walk(tree,(tree_walk_action) test_when_accessed, + (void*) &access_param,left_root_right); + if (!access_param.found) + return 1; /* No open file that is possibly to close */ + if (mi_close(access_param.found->isam)) + return 1; + access_param.found->closed=1; + return 0; +} + + +static int reopen_closed_file(TREE *tree, struct file_info *fileinfo) +{ + char name[FN_REFLEN]; + if (close_some_file(tree)) + return 1; /* No file to close */ + strmov(name,fileinfo->show_name); + if (fileinfo->id > 1) + *strrchr(name,'<')='\0'; /* Remove "<id>" */ + + if (!(fileinfo->isam= mi_open(name,O_RDWR,HA_OPEN_WAIT_IF_LOCKED))) + return 1; + fileinfo->closed=0; + re_open_count++; + return 0; +} + + /* Try to find record with uniq key */ + +static int find_record_with_key(struct file_info *file_info, uchar *record) +{ + uint key; + MI_INFO *info=file_info->isam; + uchar tmp_key[HA_MAX_KEY_BUFF]; + + for (key=0 ; key < info->s->base.keys ; key++) + { + if (mi_is_key_active(info->s->state.key_map, key) && + info->s->keyinfo[key].flag & HA_NOSAME) + { + (void) _mi_make_key(info,key,tmp_key,record,0L); + return mi_rkey(info,file_info->record,(int) key,tmp_key,0, + HA_READ_KEY_EXACT); + } + } + return 1; +} + + +static void printf_log(const char *format,...) +{ + char llbuff[21]; + va_list args; + va_start(args,format); + if (verbose > 2) + printf("%9s:",llstr(isamlog_filepos,llbuff)); + if (verbose > 1) + printf("%5ld ",isamlog_process); /* Write process number */ + (void) vprintf((char*) format,args); + putchar('\n'); + va_end(args); +} + + +static my_bool cmp_filename(struct file_info *file_info, char * name) +{ + if (!file_info) + return 1; + return strcmp(file_info->name,name) ? 1 : 0; +} + +#include "mi_extrafunc.h" diff --git a/storage/myisam/myisampack.c b/storage/myisam/myisampack.c new file mode 100644 index 00000000..d6cd9334 --- /dev/null +++ b/storage/myisam/myisampack.c @@ -0,0 +1,3237 @@ +/* Copyright (c) 2000, 2013, Oracle and/or its affiliates + Copyright (c) 2009, 2013, Monty Program Ab. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Pack MyISAM file */ + +#ifndef USE_MY_FUNC +#define USE_MY_FUNC /* We need at least my_malloc */ +#endif + +#include "myisamdef.h" +#include "my_default.h" +#include <queues.h> +#include <my_tree.h> +#include "mysys_err.h" +#ifndef __GNU_LIBRARY__ +#define __GNU_LIBRARY__ /* Skip warnings in getopt.h */ +#endif +#include <my_getopt.h> +#include <assert.h> + +#if SIZEOF_LONG_LONG > 4 +#define BITS_SAVED 64 +#else +#define BITS_SAVED 32 +#endif + +#define IS_OFFSET ((uint) 32768) /* Bit if offset or char in tree */ +#define HEAD_LENGTH 32 +#define ALLOWED_JOIN_DIFF 256 /* Diff allowed to join trees */ + +#define DATA_TMP_EXT ".TMD" +#define OLD_EXT ".OLD" +#define FRM_EXT ".frm" +#define WRITE_COUNT MY_HOW_OFTEN_TO_WRITE + +struct st_file_buffer { + File file; + uchar *buffer,*pos,*end; + my_off_t pos_in_file; + int bits; + ulonglong bitbucket; +}; + +struct st_huff_tree; +struct st_huff_element; + +typedef struct st_huff_counts { + uint field_length,max_zero_fill; + uint pack_type; + uint max_end_space,max_pre_space,length_bits,min_space; + ulong max_length; + enum en_fieldtype field_type; + struct st_huff_tree *tree; /* Tree for field */ + my_off_t counts[256]; + my_off_t end_space[8]; + my_off_t pre_space[8]; + my_off_t tot_end_space,tot_pre_space,zero_fields,empty_fields,bytes_packed; + TREE int_tree; /* Tree for detecting distinct column values. */ + uchar *tree_buff; /* Column values, 'field_length' each. */ + uchar *tree_pos; /* Points to end of column values in 'tree_buff'. */ +} HUFF_COUNTS; + +typedef struct st_huff_element HUFF_ELEMENT; + +/* + WARNING: It is crucial for the optimizations in calc_packed_length() + that 'count' is the first element of 'HUFF_ELEMENT'. +*/ +struct st_huff_element { + my_off_t count; + union un_element { + struct st_nod { + HUFF_ELEMENT *left,*right; + } nod; + struct st_leaf { + HUFF_ELEMENT *null; + uint element_nr; /* Number of element */ + } leaf; + } a; +}; + + +typedef struct st_huff_tree { + HUFF_ELEMENT *root,*element_buffer; + HUFF_COUNTS *counts; + uint tree_number; + uint elements; + my_off_t bytes_packed; + uint tree_pack_length; + uint min_chr,max_chr,char_bits,offset_bits,max_offset,height; + ulonglong *code; + uchar *code_len; +} HUFF_TREE; + + +typedef struct st_isam_mrg { + MI_INFO **file,**current,**end; + uint free_file; + uint count; + uint min_pack_length; /* These are used by packed data */ + uint max_pack_length; + uint ref_length; + uint max_blob_length; + my_off_t records; + /* true if at least one source file has at least one disabled index */ + my_bool src_file_has_indexes_disabled; +} PACK_MRG_INFO; + + +extern int main(int argc,char * *argv); +static void get_options(int *argc,char ***argv); +static MI_INFO *open_isam_file(char *name,int mode); +static my_bool open_isam_files(PACK_MRG_INFO *mrg,char **names,uint count); +static int compress(PACK_MRG_INFO *file,char *join_name); +static int create_dest_frm(char *source_table, char *dest_table); +static HUFF_COUNTS *init_huff_count(MI_INFO *info,my_off_t records); +static void free_counts_and_tree_and_queue(HUFF_TREE *huff_trees, + uint trees, + HUFF_COUNTS *huff_counts, + uint fields); +static int compare_tree(void* cmp_arg __attribute__((unused)), + const uchar *s,const uchar *t); +static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts); +static void check_counts(HUFF_COUNTS *huff_counts,uint trees, + my_off_t records); +static int test_space_compress(HUFF_COUNTS *huff_counts,my_off_t records, + uint max_space_length,my_off_t *space_counts, + my_off_t tot_space_count, + enum en_fieldtype field_type); +static HUFF_TREE* make_huff_trees(HUFF_COUNTS *huff_counts,uint trees); +static int make_huff_tree(HUFF_TREE *tree,HUFF_COUNTS *huff_counts); +static int compare_huff_elements(void *not_used, uchar *a,uchar *b); +static int save_counts_in_queue(uchar *key,element_count count, + HUFF_TREE *tree); +static my_off_t calc_packed_length(HUFF_COUNTS *huff_counts,uint flag); +static uint join_same_trees(HUFF_COUNTS *huff_counts,uint trees); +static int make_huff_decode_table(HUFF_TREE *huff_tree,uint trees); +static void make_traverse_code_tree(HUFF_TREE *huff_tree, + HUFF_ELEMENT *element,uint size, + ulonglong code); +static int write_header(PACK_MRG_INFO *isam_file, uint header_length,uint trees, + my_off_t tot_elements,my_off_t filelength); +static void write_field_info(HUFF_COUNTS *counts, uint fields,uint trees); +static my_off_t write_huff_tree(HUFF_TREE *huff_tree,uint trees); +static uint *make_offset_code_tree(HUFF_TREE *huff_tree, + HUFF_ELEMENT *element, + uint *offset); +static uint max_bit(uint value); +static int compress_isam_file(PACK_MRG_INFO *file,HUFF_COUNTS *huff_counts); +static char *make_new_name(char *new_name,char *old_name); +static char *make_old_name(char *new_name,char *old_name); +static void init_file_buffer(File file,pbool read_buffer); +static int flush_buffer(ulong neaded_length); +static void end_file_buffer(void); +static void write_bits(ulonglong value, uint bits); +static void flush_bits(void); +static int save_state(MI_INFO *isam_file,PACK_MRG_INFO *mrg,my_off_t new_length, + ha_checksum crc); +static int save_state_mrg(File file,PACK_MRG_INFO *isam_file,my_off_t new_length, + ha_checksum crc); +static int mrg_close(PACK_MRG_INFO *mrg); +static int mrg_rrnd(PACK_MRG_INFO *info,uchar *buf); +static void mrg_reset(PACK_MRG_INFO *mrg); +#if !defined(DBUG_OFF) +static void fakebigcodes(HUFF_COUNTS *huff_counts, HUFF_COUNTS *end_count); +static int fakecmp(my_off_t **count1, my_off_t **count2); +#endif + + +static int error_on_write=0,test_only=0,verbose=0,silent=0, + write_loop=0,force_pack=0, isamchk_neaded=0; +static int tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL; +static my_bool backup, opt_wait; +/* + tree_buff_length is somewhat arbitrary. The bigger it is the better + the chance to win in terms of compression factor. On the other hand, + this table becomes part of the compressed file header. And its length + is coded with 16 bits in the header. Hence the limit is 2**16 - 1. +*/ +static uint tree_buff_length= 65536 - MALLOC_OVERHEAD; +static char tmp_dir[FN_REFLEN]={0},*join_table; +static my_off_t intervall_length; +static ha_checksum glob_crc; +static struct st_file_buffer file_buffer; +static QUEUE queue; +static HUFF_COUNTS *global_count; +static char zero_string[]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; +static const char *load_default_groups[]= { "myisampack",0 }; + + /* The main program */ + +int main(int argc, char **argv) +{ + int error,ok; + PACK_MRG_INFO merge; + char **default_argv; + MY_INIT(argv[0]); + + load_defaults_or_exit("my", load_default_groups, &argc, &argv); + default_argv= argv; + get_options(&argc,&argv); + + error=ok=isamchk_neaded=0; + if (join_table) + { + /* + Join files into one and create FRM file for the compressed table only if + the compression succeeds + */ + if (open_isam_files(&merge,argv,(uint) argc) || + compress(&merge, join_table) || create_dest_frm(argv[0], join_table)) + error=1; + } + else while (argc--) + { + MI_INFO *isam_file; + if (!(isam_file=open_isam_file(*argv++,O_RDWR))) + error=1; + else + { + merge.file= &isam_file; + merge.current=0; + merge.free_file=0; + merge.count=1; + if (compress(&merge,0)) + error=1; + else + ok=1; + } + } + if (ok && isamchk_neaded && !silent) + puts("Remember to run myisamchk -rq on compressed tables"); + (void) fflush(stdout); + (void) fflush(stderr); + free_defaults(default_argv); + my_end(verbose ? MY_CHECK_ERROR | MY_GIVE_INFO : MY_CHECK_ERROR); + exit(error ? 2 : 0); +#ifndef _lint + return 0; /* No compiler warning */ +#endif +} + +enum options_mp {OPT_CHARSETS_DIR_MP=256}; + +static struct my_option my_long_options[] = +{ + {"backup", 'b', "Make a backup of the table as table_name.OLD.", + &backup, &backup, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"character-sets-dir", OPT_CHARSETS_DIR_MP, + "Directory where character sets are.", (char**) &charsets_dir, + (char**) &charsets_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"debug", '#', "Output debug log. Often this is 'd:t:o,filename'.", + 0, 0, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + {"force", 'f', + "Force packing of table even if it gets bigger or if tempfile exists.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"join", 'j', + "Join all given tables into 'new_table_name'. All tables MUST have identical layouts.", + &join_table, &join_table, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, + 0, 0, 0}, + {"help", '?', "Display this help and exit.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"silent", 's', "Be more silent.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"tmpdir", 'T', "Use temporary directory to store temporary table.", + 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"test", 't', "Don't pack table, only test packing it.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"verbose", 'v', "Write info about progress and packing result. Use many -v for more verbosity!", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"version", 'V', "Output version information and exit.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"wait", 'w', "Wait and retry if table is in use.", &opt_wait, + &opt_wait, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + + +static void print_version(void) +{ + printf("%s Ver 1.23 for %s on %s\n", + my_progname, SYSTEM_TYPE, MACHINE_TYPE); +} + + +static void usage(void) +{ + print_version(); + puts("Copyright 2002-2008 MySQL AB, 2008 Sun Microsystems, Inc."); + puts("This software comes with ABSOLUTELY NO WARRANTY. This is free software,"); + puts("and you are welcome to modify and redistribute it under the GPL license\n"); + + puts("Pack a MyISAM-table to take much less space."); + puts("Keys are not updated, you must run myisamchk -rq on the index (.MYI) file"); + puts("afterwards to update the keys."); + puts("You should give the .MYI file as the filename argument."); + + printf("\nUsage: %s [OPTIONS] filename...\n", my_progname); + my_print_help(my_long_options); + print_defaults("my", load_default_groups); + my_print_variables(my_long_options); +} + + +static my_bool +get_one_option(const struct my_option *opt, + const char *argument, + const char *filename __attribute__((unused))) +{ + uint length; + + switch(opt->id) { + case 'f': + force_pack= 1; + tmpfile_createflag= O_RDWR | O_TRUNC; + break; + case 's': + write_loop= verbose= 0; + silent= 1; + break; + case 't': + test_only= 1; + /* Avoid to reset 'verbose' if it was already set > 1. */ + if (! verbose) + verbose= 1; + break; + case 'T': + length= (uint) (strmov(tmp_dir, argument) - tmp_dir); + if (length != dirname_length(tmp_dir)) + { + tmp_dir[length]=FN_LIBCHAR; + tmp_dir[length+1]=0; + } + break; + case 'v': + verbose++; /* Allow for selecting the level of verbosity. */ + silent= 0; + break; + case '#': + DBUG_PUSH(argument ? argument : "d:t:o"); + break; + case 'V': + print_version(); + exit(0); + case 'I': + case '?': + usage(); + exit(0); + } + return 0; +} + + /* reads options */ + /* Initiates DEBUG - but no debugging here ! */ + +static void get_options(int *argc,char ***argv) +{ + int ho_error; + + my_progname= argv[0][0]; + if (isatty(fileno(stdout))) + write_loop=1; + + if ((ho_error=handle_options(argc, argv, my_long_options, get_one_option))) + exit(ho_error); + + if (!*argc) + { + usage(); + exit(1); + } + if (join_table) + { + backup=0; /* Not needed */ + tmp_dir[0]=0; + } + return; +} + + +static MI_INFO *open_isam_file(char *name,int mode) +{ + MI_INFO *isam_file; + MYISAM_SHARE *share; + DBUG_ENTER("open_isam_file"); + + if (!(isam_file=mi_open(name,mode, + (opt_wait ? HA_OPEN_WAIT_IF_LOCKED : + HA_OPEN_ABORT_IF_LOCKED)))) + { + (void) fprintf(stderr, "%s gave error %d on open\n", name, my_errno); + DBUG_RETURN(0); + } + share=isam_file->s; + if (share->options & HA_OPTION_COMPRESS_RECORD && !join_table) + { + if (!force_pack) + { + (void) fprintf(stderr, "%s is already compressed\n", name); + (void) mi_close(isam_file); + DBUG_RETURN(0); + } + if (verbose) + puts("Recompressing already compressed table"); + share->options&= ~HA_OPTION_READ_ONLY_DATA; /* We are modifing it */ + + /* We want to use the new checksums if we have null fields */ + if (share->has_null_fields) + share->options|= HA_OPTION_NULL_FIELDS; + + } + if (! force_pack && share->state.state.records != 0 && + (share->state.state.records <= 1 || + share->state.state.data_file_length < 1024)) + { + (void) fprintf(stderr, "%s is too small to compress\n", name); + (void) mi_close(isam_file); + DBUG_RETURN(0); + } + (void) mi_lock_database(isam_file,F_WRLCK); + DBUG_RETURN(isam_file); +} + + +static my_bool open_isam_files(PACK_MRG_INFO *mrg, char **names, uint count) +{ + uint i,j; + mrg->count=0; + mrg->current=0; + mrg->file=(MI_INFO**) my_malloc(PSI_NOT_INSTRUMENTED, sizeof(MI_INFO*)*count,MYF(MY_FAE)); + mrg->free_file=1; + mrg->src_file_has_indexes_disabled= 0; + for (i=0; i < count ; i++) + { + if (!(mrg->file[i]=open_isam_file(names[i],O_RDONLY))) + goto error; + + mrg->src_file_has_indexes_disabled|= + ! mi_is_all_keys_active(mrg->file[i]->s->state.key_map, + mrg->file[i]->s->base.keys); + } + /* Check that files are identical */ + for (j=0 ; j < count-1 ; j++) + { + MI_COLUMNDEF *m1,*m2,*end; + if (mrg->file[j]->s->base.reclength != mrg->file[j+1]->s->base.reclength || + mrg->file[j]->s->base.fields != mrg->file[j+1]->s->base.fields) + goto diff_file; + m1=mrg->file[j]->s->rec; + end=m1+mrg->file[j]->s->base.fields; + m2=mrg->file[j+1]->s->rec; + for ( ; m1 != end ; m1++,m2++) + { + if (m1->type != m2->type || m1->length != m2->length) + goto diff_file; + } + } + mrg->count=count; + return 0; + + diff_file: + (void) fprintf(stderr, "%s: Tables '%s' and '%s' are not identical\n", + my_progname, names[j], names[j+1]); + error: + while (i--) + mi_close(mrg->file[i]); + my_free(mrg->file); + return 1; +} + + +static int compress(PACK_MRG_INFO *mrg,char *result_table) +{ + int error; + File new_file,join_isam_file; + MI_INFO *isam_file; + MYISAM_SHARE *share; + char org_name[FN_REFLEN],new_name[FN_REFLEN],temp_name[FN_REFLEN]; + uint i,header_length,fields,trees,used_trees; + my_off_t old_length,new_length,tot_elements; + HUFF_COUNTS *huff_counts; + HUFF_TREE *huff_trees; + DBUG_ENTER("compress"); + + isam_file=mrg->file[0]; /* Take this as an example */ + share=isam_file->s; + new_file=join_isam_file= -1; + trees=fields=0; + huff_trees=0; + huff_counts=0; + + /* Create temporary or join file */ + + if (backup) + (void) fn_format(org_name,isam_file->filename,"",MI_NAME_DEXT,2); + else + (void) fn_format(org_name,isam_file->filename,"",MI_NAME_DEXT,2+4+16); + if (!test_only && result_table) + { + /* Make a new indexfile based on first file in list */ + uint length; + uchar *buff; + strmov(org_name,result_table); /* Fix error messages */ + (void) fn_format(new_name,result_table,"",MI_NAME_IEXT,2); + if ((join_isam_file=my_create(new_name,0,tmpfile_createflag,MYF(MY_WME))) + < 0) + goto err; + length=(uint) share->base.keystart; + if (!(buff= (uchar*) my_malloc(PSI_NOT_INSTRUMENTED, length,MYF(MY_WME)))) + goto err; + if (my_pread(share->kfile,buff,length,0L,MYF(MY_WME | MY_NABP)) || + my_write(join_isam_file,buff,length, + MYF(MY_WME | MY_NABP | MY_WAIT_IF_FULL))) + { + my_free(buff); + goto err; + } + my_free(buff); + (void) fn_format(new_name,result_table,"",MI_NAME_DEXT,2); + } + else if (!tmp_dir[0]) + (void) make_new_name(new_name,org_name); + else + (void) fn_format(new_name,org_name,tmp_dir,DATA_TMP_EXT,1+2+4); + if (!test_only && + (new_file=my_create(new_name,0,tmpfile_createflag,MYF(MY_WME))) < 0) + goto err; + + /* Start calculating statistics */ + + mrg->records=0; + for (i=0 ; i < mrg->count ; i++) + mrg->records+=mrg->file[i]->s->state.state.records; + + DBUG_PRINT("info", ("Compressing %s: (%lu records)", + result_table ? new_name : org_name, + (ulong) mrg->records)); + if (write_loop || verbose) + { + printf("Compressing %s: (%lu records)\n", + result_table ? new_name : org_name, (ulong) mrg->records); + } + trees=fields=share->base.fields; + huff_counts=init_huff_count(isam_file,mrg->records); + + /* + Read the whole data file(s) for statistics. + */ + DBUG_PRINT("info", ("- Calculating statistics")); + if (write_loop || verbose) + printf("- Calculating statistics\n"); + if (get_statistic(mrg,huff_counts)) + goto err; + + old_length=0; + for (i=0; i < mrg->count ; i++) + old_length+= (mrg->file[i]->s->state.state.data_file_length - + mrg->file[i]->s->state.state.empty); + + /* + Create a global priority queue in preparation for making + temporary Huffman trees. + */ + if (init_queue(&queue, 256, 0, 0, compare_huff_elements, 0, 0, 0)) + goto err; + + /* + Check each column if we should use pre-space-compress, end-space- + compress, empty-field-compress or zero-field-compress. + */ + check_counts(huff_counts,fields,mrg->records); + + /* + Build a Huffman tree for each column. + */ + huff_trees=make_huff_trees(huff_counts,trees); + + /* + If the packed lengths of combined columns is less then the sum of + the non-combined columns, then create common Huffman trees for them. + We do this only for byte compressed columns, not for distinct values + compressed columns. + */ + if ((int) (used_trees=join_same_trees(huff_counts,trees)) < 0) + goto err; + + /* + Assign codes to all byte or column values. + */ + if (make_huff_decode_table(huff_trees,fields)) + goto err; + + /* Prepare a file buffer. */ + init_file_buffer(new_file,0); + + /* + Reserve space in the target file for the fixed compressed file header. + */ + file_buffer.pos_in_file=HEAD_LENGTH; + if (! test_only) + my_seek(new_file,file_buffer.pos_in_file,MY_SEEK_SET,MYF(0)); + + /* + Write field infos: field type, pack type, length bits, tree number. + */ + write_field_info(huff_counts,fields,used_trees); + + /* + Write decode trees. + */ + if (!(tot_elements=write_huff_tree(huff_trees,trees))) + goto err; + + /* + Calculate the total length of the compression info header. + This includes the fixed compressed file header, the column compression + type descriptions, and the decode trees. + */ + header_length=(uint) file_buffer.pos_in_file+ + (uint) (file_buffer.pos-file_buffer.buffer); + + /* + Compress the source file into the target file. + */ + DBUG_PRINT("info", ("- Compressing file")); + if (write_loop || verbose) + printf("- Compressing file\n"); + error=compress_isam_file(mrg,huff_counts); + new_length=file_buffer.pos_in_file; + if (!error && !test_only) + { + uchar buff[MEMMAP_EXTRA_MARGIN]; /* End marginal for memmap */ + bzero(buff,sizeof(buff)); + error=my_write(file_buffer.file,buff,sizeof(buff), + MYF(MY_WME | MY_NABP | MY_WAIT_IF_FULL)) != 0; + } + + /* + Write the fixed compressed file header. + */ + if (!error) + error=write_header(mrg,header_length,used_trees,tot_elements, + new_length); + + /* Flush the file buffer. */ + end_file_buffer(); + + /* Display statistics. */ + DBUG_PRINT("info", ("Min record length: %6d Max length: %6d " + "Mean total length: %6ld\n", + mrg->min_pack_length, mrg->max_pack_length, + (ulong) (mrg->records ? (new_length/mrg->records) : 0))); + if (verbose && mrg->records) + printf("Min record length: %6d Max length: %6d " + "Mean total length: %6ld\n", mrg->min_pack_length, + mrg->max_pack_length, (ulong) (new_length/mrg->records)); + + /* Close source and target file. */ + if (!test_only) + { + error|=my_close(new_file,MYF(MY_WME)); + if (!result_table) + { + error|=my_close(isam_file->dfile,MYF(MY_WME)); + isam_file->dfile= -1; /* Tell mi_close file is closed */ + } + } + + /* Cleanup. */ + free_counts_and_tree_and_queue(huff_trees,trees,huff_counts,fields); + if (! test_only && ! error) + { + if (result_table) + { + error=save_state_mrg(join_isam_file,mrg,new_length,glob_crc); + } + else + { + if (backup) + { + if (my_rename(org_name,make_old_name(temp_name,isam_file->filename), + MYF(MY_WME))) + error=1; + else + { + if (tmp_dir[0]) + error=my_copy(new_name,org_name,MYF(MY_WME)); + else + error=my_rename(new_name,org_name,MYF(MY_WME)); + if (!error) + { + (void) my_copystat(temp_name,org_name,MYF(MY_COPYTIME)); + if (tmp_dir[0]) + (void) my_delete(new_name,MYF(MY_WME)); + } + } + } + else + { + if (tmp_dir[0]) + { + error=my_copy(new_name,org_name, + MYF(MY_WME | MY_HOLD_ORIGINAL_MODES | MY_COPYTIME)); + if (!error) + (void) my_delete(new_name,MYF(MY_WME)); + } + else + error=my_redel(org_name, new_name, 0, MYF(MY_WME | MY_COPYTIME)); + } + if (! error) + error=save_state(isam_file,mrg,new_length,glob_crc); + } + } + error|=mrg_close(mrg); + if (join_isam_file >= 0) + error|=my_close(join_isam_file,MYF(MY_WME)); + if (error) + { + (void) fprintf(stderr, "Aborting: %s is not compressed\n", org_name); + (void) my_delete(new_name,MYF(MY_WME)); + DBUG_RETURN(-1); + } + if (write_loop || verbose) + { + if (old_length) + printf("%.4g%% \n", + (((longlong) (old_length - new_length)) * 100.0 / + (longlong) old_length)); + else + puts("Empty file saved in compressed format"); + } + DBUG_RETURN(0); + + err: + free_counts_and_tree_and_queue(huff_trees,trees,huff_counts,fields); + if (new_file >= 0) + (void) my_close(new_file,MYF(0)); + if (join_isam_file >= 0) + (void) my_close(join_isam_file,MYF(0)); + mrg_close(mrg); + (void) fprintf(stderr, "Aborted: %s is not compressed\n", org_name); + DBUG_RETURN(-1); +} + + +/** + Create FRM for the destination table for --join operation + Copy the first table FRM as the destination table FRM file. Doing so + will help the mysql server to recognize the newly created table. + See Bug#36573. + + @param source_table Name of the source table + @param dest_table Name of the destination table + @retval 0 Successful copy operation + + @note We always return 0 because we don't want myisampack to report error + even if the copy operation fails. +*/ + +static int create_dest_frm(char *source_table, char *dest_table) +{ + char source_name[FN_REFLEN], dest_name[FN_REFLEN]; + + DBUG_ENTER("create_dest_frm"); + + (void) fn_format(source_name, source_table, + "", FRM_EXT, MY_UNPACK_FILENAME | MY_RESOLVE_SYMLINKS); + (void) fn_format(dest_name, dest_table, + "", FRM_EXT, MY_UNPACK_FILENAME | MY_RESOLVE_SYMLINKS); + /* + Error messages produced by my_copy() are suppressed as this + is not vital for --join operation. User shouldn't see any error messages + like "source file frm not found" and "unable to create destination frm + file. So we don't pass the flag MY_WME -Write Message on Error to + my_copy() + */ + (void) my_copy(source_name, dest_name, MYF(MY_DONT_OVERWRITE_FILE)); + + DBUG_RETURN(0); +} + + + /* Init a huff_count-struct for each field and init it */ + +static HUFF_COUNTS *init_huff_count(MI_INFO *info,my_off_t records) +{ + reg2 uint i; + reg1 HUFF_COUNTS *count; + if ((count = (HUFF_COUNTS*) my_malloc(PSI_NOT_INSTRUMENTED, info->s->base.fields* + sizeof(HUFF_COUNTS), + MYF(MY_ZEROFILL | MY_WME)))) + { + for (i=0 ; i < info->s->base.fields ; i++) + { + enum en_fieldtype type; + count[i].field_length=info->s->rec[i].length; + type= count[i].field_type= (enum en_fieldtype) info->s->rec[i].type; + if (type == FIELD_INTERVALL || + type == FIELD_CONSTANT || + type == FIELD_ZERO) + type = FIELD_NORMAL; + if (count[i].field_length <= 8 && + (type == FIELD_NORMAL || + type == FIELD_SKIP_ZERO)) + count[i].max_zero_fill= count[i].field_length; + /* + For every column initialize a tree, which is used to detect distinct + column values. 'int_tree' works together with 'tree_buff' and + 'tree_pos'. It's keys are implemented by pointers into 'tree_buff'. + This is accomplished by '-1' as the element size. + */ + init_tree(&count[i].int_tree,0,0,-1,(qsort_cmp2) compare_tree, NULL, + NULL, MYF(0)); + if (records && type != FIELD_BLOB && type != FIELD_VARCHAR) + count[i].tree_pos=count[i].tree_buff = + my_malloc(PSI_NOT_INSTRUMENTED, count[i].field_length > 1 ? tree_buff_length : 2, + MYF(MY_WME)); + } + } + return count; +} + + + /* Free memory used by counts and trees */ + +static void free_counts_and_tree_and_queue(HUFF_TREE *huff_trees, uint trees, + HUFF_COUNTS *huff_counts, + uint fields) +{ + register uint i; + + if (huff_trees) + { + for (i=0 ; i < trees ; i++) + { + if (huff_trees[i].element_buffer) + my_free(huff_trees[i].element_buffer); + if (huff_trees[i].code) + my_free(huff_trees[i].code); + } + my_free(huff_trees); + } + if (huff_counts) + { + for (i=0 ; i < fields ; i++) + { + if (huff_counts[i].tree_buff) + { + my_free(huff_counts[i].tree_buff); + delete_tree(&huff_counts[i].int_tree, 0); + } + } + my_free(huff_counts); + } + delete_queue(&queue); /* This is safe to free */ + return; +} + + /* Read through old file and gather some statistics */ + +static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts) +{ + int error; + uint length; + ulong reclength,max_blob_length; + uchar *record,*pos,*next_pos,*end_pos,*start_pos; + ha_rows record_count; + my_bool static_row_size; + HUFF_COUNTS *count,*end_count; + TREE_ELEMENT *element; + DBUG_ENTER("get_statistic"); + + reclength=mrg->file[0]->s->base.reclength; + record=(uchar*) my_alloca(reclength); + end_count=huff_counts+mrg->file[0]->s->base.fields; + record_count=0; glob_crc=0; + max_blob_length=0; + + /* Check how to calculate checksum */ + static_row_size=1; + for (count=huff_counts ; count < end_count ; count++) + { + if (count->field_type == FIELD_BLOB || + count->field_type == FIELD_VARCHAR) + { + static_row_size=0; + break; + } + } + + mrg_reset(mrg); + while ((error=mrg_rrnd(mrg,record)) != HA_ERR_END_OF_FILE) + { + ulong tot_blob_length=0; + if (! error) + { + /* glob_crc is a checksum over all bytes of all records. */ + if (static_row_size) + glob_crc+=mi_static_checksum(mrg->file[0],record); + else + glob_crc+=mi_checksum(mrg->file[0],record); + + /* Count the incidence of values separately for every column. */ + for (pos=record,count=huff_counts ; + count < end_count ; + count++, + pos=next_pos) + { + next_pos=end_pos=(start_pos=pos)+count->field_length; + + /* + Put the whole column value in a tree if there is room for it. + 'int_tree' is used to quickly check for duplicate values. + 'tree_buff' collects as many distinct column values as + possible. If the field length is > 1, it is tree_buff_length, + else 2 bytes. Each value is 'field_length' bytes big. If there + are more distinct column values than fit into the buffer, we + give up with this tree. BLOBs and VARCHARs do not have a + tree_buff as it can only be used with fixed length columns. + For the special case of field length == 1, we handle only the + case that there is only one distinct value in the table(s). + Otherwise, we can have a maximum of 256 distinct values. This + is then handled by the normal Huffman tree build. + + Another limit for collecting distinct column values is the + number of values itself. Since we would need to build a + Huffman tree for the values, we are limited by the 'IS_OFFSET' + constant. This constant expresses a bit which is used to + determine if a tree element holds a final value or an offset + to a child element. Hence, all values and offsets need to be + smaller than 'IS_OFFSET'. A tree element is implemented with + two integer values, one for the left branch and one for the + right branch. For the extreme case that the first element + points to the last element, the number of integers in the tree + must be less or equal to IS_OFFSET. So the number of elements + must be less or equal to IS_OFFSET / 2. + + WARNING: At first, we insert a pointer into the record buffer + as the key for the tree. If we got a new distinct value, which + is really inserted into the tree, instead of being counted + only, we will copy the column value from the record buffer to + 'tree_buff' and adjust the key pointer of the tree accordingly. + */ + if (count->tree_buff) + { + global_count=count; + if (!(element=tree_insert(&count->int_tree,pos, 0, + count->int_tree.custom_arg)) || + (element->count == 1 && + (count->tree_buff + tree_buff_length < + count->tree_pos + count->field_length)) || + (count->int_tree.elements_in_tree > IS_OFFSET / 2) || + (count->field_length == 1 && + count->int_tree.elements_in_tree > 1)) + { + delete_tree(&count->int_tree, 0); + my_free(count->tree_buff); + count->tree_buff=0; + } + else + { + /* + If tree_insert() succeeds, it either creates a new element + or increments the counter of an existing element. + */ + if (element->count == 1) + { + /* Copy the new column value into 'tree_buff'. */ + memcpy(count->tree_pos,pos,(size_t) count->field_length); + /* Adjust the key pointer in the tree. */ + tree_set_pointer(element,count->tree_pos); + /* Point behind the last column value so far. */ + count->tree_pos+=count->field_length; + } + } + } + + /* Save character counters and space-counts and zero-field-counts */ + if (count->field_type == FIELD_NORMAL || + count->field_type == FIELD_SKIP_ENDSPACE) + { + /* Ignore trailing space. */ + for ( ; end_pos > pos ; end_pos--) + if (end_pos[-1] != ' ') + break; + /* Empty fields are just counted. Go to the next record. */ + if (end_pos == pos) + { + count->empty_fields++; + count->max_zero_fill=0; + continue; + } + /* + Count the total of all trailing spaces and the number of + short trailing spaces. Remember the longest trailing space. + */ + length= (uint) (next_pos-end_pos); + count->tot_end_space+=length; + if (length < 8) + count->end_space[length]++; + if (count->max_end_space < length) + count->max_end_space = length; + } + + if (count->field_type == FIELD_NORMAL || + count->field_type == FIELD_SKIP_PRESPACE) + { + /* Ignore leading space. */ + for (pos=start_pos; pos < end_pos ; pos++) + if (pos[0] != ' ') + break; + /* Empty fields are just counted. Go to the next record. */ + if (end_pos == pos) + { + count->empty_fields++; + count->max_zero_fill=0; + continue; + } + /* + Count the total of all leading spaces and the number of + short leading spaces. Remember the longest leading space. + */ + length= (uint) (pos-start_pos); + count->tot_pre_space+=length; + if (length < 8) + count->pre_space[length]++; + if (count->max_pre_space < length) + count->max_pre_space = length; + } + + /* Calculate pos, end_pos, and max_length for variable length fields. */ + if (count->field_type == FIELD_BLOB) + { + uint field_length=count->field_length -portable_sizeof_char_ptr; + ulong blob_length= _mi_calc_blob_length(field_length, start_pos); + memcpy(&pos, start_pos+field_length, sizeof(char*)); + end_pos=pos+blob_length; + tot_blob_length+=blob_length; + set_if_bigger(count->max_length,blob_length); + } + else if (count->field_type == FIELD_VARCHAR) + { + uint pack_length= HA_VARCHAR_PACKLENGTH(count->field_length-1); + length= (pack_length == 1 ? (uint) *(uchar*) start_pos : + uint2korr(start_pos)); + pos= start_pos+pack_length; + end_pos= pos+length; + set_if_bigger(count->max_length,length); + } + + /* Evaluate 'max_zero_fill' for short fields. */ + if (count->field_length <= 8 && + (count->field_type == FIELD_NORMAL || + count->field_type == FIELD_SKIP_ZERO)) + { + uint i; + /* Zero fields are just counted. Go to the next record. */ + if (!memcmp((uchar*) start_pos,zero_string,count->field_length)) + { + count->zero_fields++; + continue; + } + /* + max_zero_fill starts with field_length. It is decreased every + time a shorter "zero trailer" is found. It is set to zero when + an empty field is found (see above). This suggests that the + variable should be called 'min_zero_fill'. + */ + for (i =0 ; i < count->max_zero_fill && ! end_pos[-1 - (int) i] ; + i++) ; + if (i < count->max_zero_fill) + count->max_zero_fill=i; + } + + /* Ignore zero fields and check fields. */ + if (count->field_type == FIELD_ZERO || + count->field_type == FIELD_CHECK) + continue; + + /* + Count the incidence of every byte value in the + significant field value. + */ + for ( ; pos < end_pos ; pos++) + count->counts[(uchar) *pos]++; + + /* Step to next field. */ + } + + if (tot_blob_length > max_blob_length) + max_blob_length=tot_blob_length; + record_count++; + if (write_loop && record_count % WRITE_COUNT == 0) + { + printf("%lu\r", (ulong) record_count); + (void) fflush(stdout); + } + } + else if (error != HA_ERR_RECORD_DELETED) + { + (void) fprintf(stderr, "Got error %d while reading rows", error); + break; + } + + /* Step to next record. */ + } + if (write_loop) + { + printf(" \r"); + (void) fflush(stdout); + } + + /* + If --debug=d,fakebigcodes is set, fake the counts to get big Huffman + codes. + */ + DBUG_EXECUTE_IF("fakebigcodes", fakebigcodes(huff_counts, end_count);); + + DBUG_PRINT("info", ("Found the following number of incidents " + "of the byte codes:")); + if (verbose >= 2) + printf("Found the following number of incidents " + "of the byte codes:\n"); + for (count= huff_counts ; count < end_count; count++) + { + uint idx; + my_off_t total_count; + char llbuf[32]; + + DBUG_PRINT("info", ("column: %3u", (uint) (count - huff_counts + 1))); + if (verbose >= 2) + printf("column: %3u\n", (uint) (count - huff_counts + 1)); + if (count->tree_buff) + { + DBUG_PRINT("info", ("number of distinct values: %u", + (uint) ((count->tree_pos - count->tree_buff) / + count->field_length))); + if (verbose >= 2) + printf("number of distinct values: %u\n", + (uint) ((count->tree_pos - count->tree_buff) / + count->field_length)); + } + total_count= 0; + for (idx= 0; idx < 256; idx++) + { + if (count->counts[idx]) + { + total_count+= count->counts[idx]; + DBUG_PRINT("info", ("counts[0x%02x]: %12s", idx, + llstr((longlong) count->counts[idx], llbuf))); + if (verbose >= 2) + printf("counts[0x%02x]: %12s\n", idx, + llstr((longlong) count->counts[idx], llbuf)); + } + } + DBUG_PRINT("info", ("total: %12s", llstr((longlong) total_count, + llbuf))); + if ((verbose >= 2) && total_count) + { + printf("total: %12s\n", + llstr((longlong) total_count, llbuf)); + } + } + + mrg->records=record_count; + mrg->max_blob_length=max_blob_length; + my_afree((uchar*) record); + DBUG_RETURN(error != HA_ERR_END_OF_FILE); +} + +static int compare_huff_elements(void *not_used __attribute__((unused)), + uchar *a, uchar *b) +{ + return *((my_off_t*) a) < *((my_off_t*) b) ? -1 : + (*((my_off_t*) a) == *((my_off_t*) b) ? 0 : 1); +} + + /* Check each tree if we should use pre-space-compress, end-space- + compress, empty-field-compress or zero-field-compress */ + +static void check_counts(HUFF_COUNTS *huff_counts, uint trees, + my_off_t records) +{ + uint space_fields,fill_zero_fields,field_count[(int) FIELD_enum_val_count]; + my_off_t old_length,new_length,length; + DBUG_ENTER("check_counts"); + + bzero((uchar*) field_count,sizeof(field_count)); + space_fields=fill_zero_fields=0; + + for (; trees-- ; huff_counts++) + { + if (huff_counts->field_type == FIELD_BLOB) + { + huff_counts->length_bits=max_bit(huff_counts->max_length); + goto found_pack; + } + else if (huff_counts->field_type == FIELD_VARCHAR) + { + huff_counts->length_bits=max_bit(huff_counts->max_length); + goto found_pack; + } + else if (huff_counts->field_type == FIELD_CHECK) + { + huff_counts->bytes_packed=0; + huff_counts->counts[0]=0; + goto found_pack; + } + + huff_counts->field_type=FIELD_NORMAL; + huff_counts->pack_type=0; + + /* Check for zero-filled records (in this column), or zero records. */ + if (huff_counts->zero_fields || ! records) + { + my_off_t old_space_count; + /* + If there are only zero filled records (in this column), + or no records at all, we are done. + */ + if (huff_counts->zero_fields == records) + { + huff_counts->field_type= FIELD_ZERO; + huff_counts->bytes_packed=0; + huff_counts->counts[0]=0; + goto found_pack; + } + /* Remember the number of significant spaces. */ + old_space_count=huff_counts->counts[' ']; + /* Add all leading and trailing spaces. */ + huff_counts->counts[' ']+= (huff_counts->tot_end_space + + huff_counts->tot_pre_space + + huff_counts->empty_fields * + huff_counts->field_length); + /* Check, what the compressed length of this would be. */ + old_length=calc_packed_length(huff_counts,0)+records/8; + /* Get the number of zero bytes. */ + length=huff_counts->zero_fields*huff_counts->field_length; + /* Add it to the counts. */ + huff_counts->counts[0]+=length; + /* Check, what the compressed length of this would be. */ + new_length=calc_packed_length(huff_counts,0); + /* If the compression without the zeroes would be shorter, we are done. */ + if (old_length < new_length && huff_counts->field_length > 1) + { + huff_counts->field_type=FIELD_SKIP_ZERO; + huff_counts->counts[0]-=length; + huff_counts->bytes_packed=old_length- records/8; + goto found_pack; + } + /* Remove the insignificant spaces, but keep the zeroes. */ + huff_counts->counts[' ']=old_space_count; + } + /* Check, what the compressed length of this column would be. */ + huff_counts->bytes_packed=calc_packed_length(huff_counts,0); + + /* + If there are enough empty records (in this column), + treating them specially may pay off. + */ + if (huff_counts->empty_fields) + { + if (huff_counts->field_length > 2 && + huff_counts->empty_fields + (records - huff_counts->empty_fields)* + (1+max_bit(MY_MAX(huff_counts->max_pre_space, + huff_counts->max_end_space))) < + records * max_bit(huff_counts->field_length)) + { + huff_counts->pack_type |= PACK_TYPE_SPACE_FIELDS; + } + else + { + length=huff_counts->empty_fields*huff_counts->field_length; + if (huff_counts->tot_end_space || ! huff_counts->tot_pre_space) + { + huff_counts->tot_end_space+=length; + huff_counts->max_end_space=huff_counts->field_length; + if (huff_counts->field_length < 8) + huff_counts->end_space[huff_counts->field_length]+= + huff_counts->empty_fields; + } + if (huff_counts->tot_pre_space) + { + huff_counts->tot_pre_space+=length; + huff_counts->max_pre_space=huff_counts->field_length; + if (huff_counts->field_length < 8) + huff_counts->pre_space[huff_counts->field_length]+= + huff_counts->empty_fields; + } + } + } + + /* + If there are enough trailing spaces (in this column), + treating them specially may pay off. + */ + if (huff_counts->tot_end_space) + { + huff_counts->counts[' ']+=huff_counts->tot_pre_space; + if (test_space_compress(huff_counts,records,huff_counts->max_end_space, + huff_counts->end_space, + huff_counts->tot_end_space,FIELD_SKIP_ENDSPACE)) + goto found_pack; + huff_counts->counts[' ']-=huff_counts->tot_pre_space; + } + + /* + If there are enough leading spaces (in this column), + treating them specially may pay off. + */ + if (huff_counts->tot_pre_space) + { + if (test_space_compress(huff_counts,records,huff_counts->max_pre_space, + huff_counts->pre_space, + huff_counts->tot_pre_space,FIELD_SKIP_PRESPACE)) + goto found_pack; + } + + found_pack: /* Found field-packing */ + + /* Test if we can use zero-fill */ + + if (huff_counts->max_zero_fill && + (huff_counts->field_type == FIELD_NORMAL || + huff_counts->field_type == FIELD_SKIP_ZERO)) + { + huff_counts->counts[0]-=huff_counts->max_zero_fill* + (huff_counts->field_type == FIELD_SKIP_ZERO ? + records - huff_counts->zero_fields : records); + huff_counts->pack_type|=PACK_TYPE_ZERO_FILL; + huff_counts->bytes_packed=calc_packed_length(huff_counts,0); + } + + /* Test if intervall-field is better */ + + if (huff_counts->tree_buff) + { + HUFF_TREE tree; + + DBUG_EXECUTE_IF("forceintervall", + huff_counts->bytes_packed= ~ (my_off_t) 0;); + tree.element_buffer=0; + if (!make_huff_tree(&tree,huff_counts) && + tree.bytes_packed+tree.tree_pack_length < huff_counts->bytes_packed) + { + if (tree.elements == 1) + huff_counts->field_type=FIELD_CONSTANT; + else + huff_counts->field_type=FIELD_INTERVALL; + huff_counts->pack_type=0; + } + else + { + my_free(huff_counts->tree_buff); + delete_tree(&huff_counts->int_tree, 0); + huff_counts->tree_buff=0; + } + if (tree.element_buffer) + my_free(tree.element_buffer); + } + if (huff_counts->pack_type & PACK_TYPE_SPACE_FIELDS) + space_fields++; + if (huff_counts->pack_type & PACK_TYPE_ZERO_FILL) + fill_zero_fields++; + field_count[huff_counts->field_type]++; + } + DBUG_PRINT("info", ("normal: %3d empty-space: %3d " + "empty-zero: %3d empty-fill: %3d", + field_count[FIELD_NORMAL],space_fields, + field_count[FIELD_SKIP_ZERO],fill_zero_fields)); + DBUG_PRINT("info", ("pre-space: %3d end-space: %3d " + "intervall-fields: %3d zero: %3d", + field_count[FIELD_SKIP_PRESPACE], + field_count[FIELD_SKIP_ENDSPACE], + field_count[FIELD_INTERVALL], + field_count[FIELD_ZERO])); + if (verbose) + printf("\nnormal: %3d empty-space: %3d " + "empty-zero: %3d empty-fill: %3d\n" + "pre-space: %3d end-space: %3d " + "intervall-fields: %3d zero: %3d\n", + field_count[FIELD_NORMAL],space_fields, + field_count[FIELD_SKIP_ZERO],fill_zero_fields, + field_count[FIELD_SKIP_PRESPACE], + field_count[FIELD_SKIP_ENDSPACE], + field_count[FIELD_INTERVALL], + field_count[FIELD_ZERO]); + DBUG_VOID_RETURN; +} + + /* Test if we can use space-compression and empty-field-compression */ + +static int +test_space_compress(HUFF_COUNTS *huff_counts, my_off_t records, + uint max_space_length, my_off_t *space_counts, + my_off_t tot_space_count, enum en_fieldtype field_type) +{ + int min_pos; + uint length_bits,i; + my_off_t space_count,min_space_count,min_pack,new_length,skip; + + length_bits=max_bit(max_space_length); + + /* Default no end_space-packing */ + space_count=huff_counts->counts[(uint) ' ']; + min_space_count= (huff_counts->counts[(uint) ' ']+= tot_space_count); + min_pack=calc_packed_length(huff_counts,0); + min_pos= -2; + huff_counts->counts[(uint) ' ']=space_count; + + /* Test with allways space-count */ + new_length=huff_counts->bytes_packed+length_bits*records/8; + if (new_length+1 < min_pack) + { + min_pos= -1; + min_pack=new_length; + min_space_count=space_count; + } + /* Test with length-flag */ + for (skip=0L, i=0 ; i < 8 ; i++) + { + if (space_counts[i]) + { + if (i) + huff_counts->counts[(uint) ' ']+=space_counts[i]; + skip+=huff_counts->pre_space[i]; + new_length=calc_packed_length(huff_counts,0)+ + (records+(records-skip)*(1+length_bits))/8; + if (new_length < min_pack) + { + min_pos=(int) i; + min_pack=new_length; + min_space_count=huff_counts->counts[(uint) ' ']; + } + } + } + + huff_counts->counts[(uint) ' ']=min_space_count; + huff_counts->bytes_packed=min_pack; + switch (min_pos) { + case -2: + return(0); /* No space-compress */ + case -1: /* Always space-count */ + huff_counts->field_type=field_type; + huff_counts->min_space=0; + huff_counts->length_bits=max_bit(max_space_length); + break; + default: + huff_counts->field_type=field_type; + huff_counts->min_space=(uint) min_pos; + huff_counts->pack_type|=PACK_TYPE_SELECTED; + huff_counts->length_bits=max_bit(max_space_length); + break; + } + return(1); /* Using space-compress */ +} + + + /* Make a huff_tree of each huff_count */ + +static HUFF_TREE* make_huff_trees(HUFF_COUNTS *huff_counts, uint trees) +{ + uint tree; + HUFF_TREE *huff_tree; + DBUG_ENTER("make_huff_trees"); + + if (!(huff_tree=(HUFF_TREE*) my_malloc(PSI_NOT_INSTRUMENTED, trees*sizeof(HUFF_TREE), + MYF(MY_WME | MY_ZEROFILL)))) + DBUG_RETURN(0); + + for (tree=0 ; tree < trees ; tree++) + { + if (make_huff_tree(huff_tree+tree,huff_counts+tree)) + { + while (tree--) + my_free(huff_tree[tree].element_buffer); + my_free(huff_tree); + DBUG_RETURN(0); + } + } + DBUG_RETURN(huff_tree); +} + +/* + Build a Huffman tree. + + SYNOPSIS + make_huff_tree() + huff_tree The Huffman tree. + huff_counts The counts. + + DESCRIPTION + Build a Huffman tree according to huff_counts->counts or + huff_counts->tree_buff. tree_buff, if non-NULL contains up to + tree_buff_length of distinct column values. In that case, whole + values can be Huffman encoded instead of single bytes. + + RETURN + 0 OK + != 0 Error +*/ + +static int make_huff_tree(HUFF_TREE *huff_tree, HUFF_COUNTS *huff_counts) +{ + uint i,found,bits_packed,first,last; + my_off_t bytes_packed; + HUFF_ELEMENT *a,*b,*new_huff_el; + + first=last=0; + if (huff_counts->tree_buff) + { + /* Calculate the number of distinct values in tree_buff. */ + found= (uint) (huff_counts->tree_pos - huff_counts->tree_buff) / + huff_counts->field_length; + first=0; last=found-1; + } + else + { + /* Count the number of byte codes found in the column. */ + for (i=found=0 ; i < 256 ; i++) + { + if (huff_counts->counts[i]) + { + if (! found++) + first=i; + last=i; + } + } + if (found < 2) + found=2; + } + + /* When using 'tree_buff' we can have more that 256 values. */ + if (queue.max_elements < found) + { + delete_queue(&queue); + if (init_queue(&queue,found, 0, 0, compare_huff_elements, 0, 0, 0)) + return -1; + } + + /* Allocate or reallocate an element buffer for the Huffman tree. */ + if (!huff_tree->element_buffer) + { + if (!(huff_tree->element_buffer= + (HUFF_ELEMENT*) my_malloc(PSI_NOT_INSTRUMENTED, found*2*sizeof(HUFF_ELEMENT),MYF(MY_WME)))) + return 1; + } + else + { + HUFF_ELEMENT *temp; + if (!(temp= + (HUFF_ELEMENT*) my_realloc(PSI_NOT_INSTRUMENTED, (uchar*) huff_tree->element_buffer, + found*2*sizeof(HUFF_ELEMENT), + MYF(MY_WME)))) + return 1; + huff_tree->element_buffer=temp; + } + + huff_counts->tree=huff_tree; + huff_tree->counts=huff_counts; + huff_tree->min_chr=first; + huff_tree->max_chr=last; + huff_tree->char_bits=max_bit(last-first); + huff_tree->offset_bits=max_bit(found-1)+1; + + if (huff_counts->tree_buff) + { + huff_tree->elements=0; + huff_tree->tree_pack_length=(1+15+16+5+5+ + (huff_tree->char_bits+1)*found+ + (huff_tree->offset_bits+1)* + (found-2)+7)/8 + + (uint) (huff_tree->counts->tree_pos- + huff_tree->counts->tree_buff); + /* + Put a HUFF_ELEMENT into the queue for every distinct column value. + + tree_walk() calls save_counts_in_queue() for every element in + 'int_tree'. This takes elements from the target trees element + buffer and places references to them into the buffer of the + priority queue. We insert in column value order, but the order is + in fact irrelevant here. We will establish the correct order + later. + */ + tree_walk(&huff_counts->int_tree, + (int (*)(void*, element_count,void*)) save_counts_in_queue, + (uchar*) huff_tree, left_root_right); + } + else + { + huff_tree->elements=found; + huff_tree->tree_pack_length=(9+9+5+5+ + (huff_tree->char_bits+1)*found+ + (huff_tree->offset_bits+1)* + (found-2)+7)/8; + /* + Put a HUFF_ELEMENT into the queue for every byte code found in the column. + + The elements are taken from the target trees element buffer. + Instead of using queue_insert(), we just place references to the + elements into the buffer of the priority queue. We insert in byte + value order, but the order is in fact irrelevant here. We will + establish the correct order later. + */ + for (i=first, found=0 ; i <= last ; i++) + { + if (huff_counts->counts[i]) + { + new_huff_el=huff_tree->element_buffer+(found++); + new_huff_el->count=huff_counts->counts[i]; + new_huff_el->a.leaf.null=0; + new_huff_el->a.leaf.element_nr=i; + queue.root[found]=(uchar*) new_huff_el; + } + } + /* + If there is only a single byte value in this field in all records, + add a second element with zero incidence. This is required to enter + the loop, which builds the Huffman tree. + */ + while (found < 2) + { + new_huff_el=huff_tree->element_buffer+(found++); + new_huff_el->count=0; + new_huff_el->a.leaf.null=0; + if (last) + new_huff_el->a.leaf.element_nr=huff_tree->min_chr=last-1; + else + new_huff_el->a.leaf.element_nr=huff_tree->max_chr=last+1; + queue.root[found]=(uchar*) new_huff_el; + } + } + + /* Make a queue from the queue buffer. */ + queue.elements=found; + + /* + Make a priority queue from the queue. Construct its index so that we + have a partially ordered tree. + */ + queue_fix(&queue); + + /* The Huffman algorithm. */ + bytes_packed=0; bits_packed=0; + for (i=1 ; i < found ; i++) + { + /* + Pop the top element from the queue (the one with the least incidence). + Popping from a priority queue includes a re-ordering of the queue, + to get the next least incidence element to the top. + */ + a=(HUFF_ELEMENT*) queue_remove_top(&queue); + /* Copy the next least incidence element */ + b=(HUFF_ELEMENT*) queue_top(&queue); + /* Get a new element from the element buffer. */ + new_huff_el=huff_tree->element_buffer+found+i; + /* The new element gets the sum of the two least incidence elements. */ + new_huff_el->count=a->count+b->count; + /* + The Huffman algorithm assigns another bit to the code for a byte + every time that bytes incidence is combined (directly or indirectly) + to a new element as one of the two least incidence elements. + This means that one more bit per incidence of that byte is required + in the resulting file. So we add the new combined incidence as the + number of bits by which the result grows. + */ + bits_packed+=(uint) (new_huff_el->count & 7); + bytes_packed+=new_huff_el->count/8; + /* The new element points to its children, lesser in left. */ + new_huff_el->a.nod.left=a; + new_huff_el->a.nod.right=b; + /* + Replace the copied top element by the new element and re-order the + queue. + */ + queue_top(&queue)= (uchar*) new_huff_el; + queue_replace_top(&queue); + } + huff_tree->root=(HUFF_ELEMENT*) queue.root[1]; + huff_tree->bytes_packed=bytes_packed+(bits_packed+7)/8; + return 0; +} + +static int compare_tree(void* cmp_arg __attribute__((unused)), + register const uchar *s, register const uchar *t) +{ + uint length; + for (length=global_count->field_length; length-- ;) + if (*s++ != *t++) + return (int) s[-1] - (int) t[-1]; + return 0; +} + +/* + Organize distinct column values and their incidences into a priority queue. + + SYNOPSIS + save_counts_in_queue() + key The column value. + count The incidence of this value. + tree The Huffman tree to be built later. + + DESCRIPTION + We use the element buffer of the targeted tree. The distinct column + values are organized in a priority queue first. The Huffman + algorithm will later organize the elements into a Huffman tree. For + the time being, we just place references to the elements into the + queue buffer. The buffer will later be organized into a priority + queue. + + RETURN + 0 + */ + +static int save_counts_in_queue(uchar *key, element_count count, + HUFF_TREE *tree) +{ + HUFF_ELEMENT *new_huff_el; + + new_huff_el=tree->element_buffer+(tree->elements++); + new_huff_el->count=count; + new_huff_el->a.leaf.null=0; + new_huff_el->a.leaf.element_nr= (uint) (key- tree->counts->tree_buff) / + tree->counts->field_length; + queue.root[tree->elements]=(uchar*) new_huff_el; + return 0; +} + + +/* + Calculate length of file if given counts should be used. + + SYNOPSIS + calc_packed_length() + huff_counts The counts for a column of the table(s). + add_tree_lenght If the decode tree length should be added. + + DESCRIPTION + We need to follow the Huffman algorithm until we know, how many bits + are required for each byte code. But we do not need the resulting + Huffman tree. Hence, we can leave out some steps which are essential + in make_huff_tree(). + + RETURN + Number of bytes required to compress this table column. +*/ + +static my_off_t calc_packed_length(HUFF_COUNTS *huff_counts, + uint add_tree_lenght) +{ + uint i,found,bits_packed,first,last; + my_off_t bytes_packed; + HUFF_ELEMENT element_buffer[256]; + DBUG_ENTER("calc_packed_length"); + + /* + WARNING: We use a small hack for efficiency: Instead of placing + references to HUFF_ELEMENTs into the queue, we just insert + references to the counts of the byte codes which appeared in this + table column. During the Huffman algorithm they are successively + replaced by references to HUFF_ELEMENTs. This works, because + HUFF_ELEMENTs have the incidence count at their beginning. + Regardless, wether the queue array contains references to counts of + type my_off_t or references to HUFF_ELEMENTs which have the count of + type my_off_t at their beginning, it always points to a count of the + same type. + + Instead of using queue_insert(), we just copy the references into + the buffer of the priority queue. We insert in byte value order, but + the order is in fact irrelevant here. We will establish the correct + order later. + */ + first=last=0; + for (i=found=0 ; i < 256 ; i++) + { + if (huff_counts->counts[i]) + { + if (! found++) + first=i; + last=i; + /* We start with root[1], which is the queues top element. */ + queue.root[found]=(uchar*) &huff_counts->counts[i]; + } + } + if (!found) + DBUG_RETURN(0); /* Empty tree */ + /* + If there is only a single byte value in this field in all records, + add a second element with zero incidence. This is required to enter + the loop, which follows the Huffman algorithm. + */ + if (found < 2) + queue.root[++found]=(uchar*) &huff_counts->counts[last ? 0 : 1]; + + /* Make a queue from the queue buffer. */ + queue.elements=found; + + bytes_packed=0; bits_packed=0; + /* Add the length of the coding table, which would become part of the file. */ + if (add_tree_lenght) + bytes_packed=(8+9+5+5+(max_bit(last-first)+1)*found+ + (max_bit(found-1)+1+1)*(found-2) +7)/8; + + /* + Make a priority queue from the queue. Construct its index so that we + have a partially ordered tree. + */ + queue_fix(&queue); + + /* The Huffman algorithm. */ + for (i=0 ; i < found-1 ; i++) + { + my_off_t *a; + my_off_t *b; + HUFF_ELEMENT *new_huff_el; + + /* + Pop the top element from the queue (the one with the least + incidence). Popping from a priority queue includes a re-ordering + of the queue, to get the next least incidence element to the top. + */ + a= (my_off_t*) queue_remove_top(&queue); + /* Copy the next least incidence element. */ + b= (my_off_t*) queue_top(&queue); + /* Create a new element in a local (automatic) buffer. */ + new_huff_el= element_buffer + i; + /* The new element gets the sum of the two least incidence elements. */ + new_huff_el->count= *a + *b; + /* + The Huffman algorithm assigns another bit to the code for a byte + every time that bytes incidence is combined (directly or indirectly) + to a new element as one of the two least incidence elements. + This means that one more bit per incidence of that byte is required + in the resulting file. So we add the new combined incidence as the + number of bits by which the result grows. + */ + bits_packed+=(uint) (new_huff_el->count & 7); + bytes_packed+=new_huff_el->count/8; + /* + Replace the copied top element by the new element and re-order the + queue. This successively replaces the references to counts by + references to HUFF_ELEMENTs. + */ + queue_top(&queue)= (uchar*) new_huff_el; + queue_replace_top(&queue); + } + DBUG_RETURN(bytes_packed+(bits_packed+7)/8); +} + + + /* Remove trees that don't give any compression */ + +static uint join_same_trees(HUFF_COUNTS *huff_counts, uint trees) +{ + uint k,tree_number; + HUFF_COUNTS count,*i,*j,*last_count; + + last_count=huff_counts+trees; + for (tree_number=0, i=huff_counts ; i < last_count ; i++) + { + if (!i->tree->tree_number) + { + i->tree->tree_number= ++tree_number; + if (i->tree_buff) + continue; /* Don't join intervall */ + for (j=i+1 ; j < last_count ; j++) + { + if (! j->tree->tree_number && ! j->tree_buff) + { + for (k=0 ; k < 256 ; k++) + count.counts[k]=i->counts[k]+j->counts[k]; + if (calc_packed_length(&count,1) <= + i->tree->bytes_packed + j->tree->bytes_packed+ + i->tree->tree_pack_length+j->tree->tree_pack_length+ + ALLOWED_JOIN_DIFF) + { + memcpy(i->counts, count.counts, + sizeof(count.counts[0])*256); + my_free(j->tree->element_buffer); + j->tree->element_buffer=0; + j->tree=i->tree; + bmove((uchar*) i->counts,(uchar*) count.counts, + sizeof(count.counts[0])*256); + if (make_huff_tree(i->tree,i)) + return (uint) -1; + } + } + } + } + } + DBUG_PRINT("info", ("Original trees: %d After join: %d", + trees, tree_number)); + if (verbose) + printf("Original trees: %d After join: %d\n", trees, tree_number); + return tree_number; /* Return trees left */ +} + + +/* + Fill in huff_tree encode tables. + + SYNOPSIS + make_huff_decode_table() + huff_tree An array of HUFF_TREE which are to be encoded. + trees The number of HUFF_TREE in the array. + + RETURN + 0 success + != 0 error +*/ + +static int make_huff_decode_table(HUFF_TREE *huff_tree, uint trees) +{ + uint elements; + for ( ; trees-- ; huff_tree++) + { + if (huff_tree->tree_number > 0) + { + elements=huff_tree->counts->tree_buff ? huff_tree->elements : 256; + if (!(huff_tree->code = + (ulonglong*) my_malloc(PSI_NOT_INSTRUMENTED, elements* + (sizeof(ulonglong) + sizeof(uchar)), + MYF(MY_WME | MY_ZEROFILL)))) + return 1; + huff_tree->code_len=(uchar*) (huff_tree->code+elements); + make_traverse_code_tree(huff_tree, huff_tree->root, + 8 * sizeof(ulonglong), 0); + } + } + return 0; +} + + +static void make_traverse_code_tree(HUFF_TREE *huff_tree, + HUFF_ELEMENT *element, + uint size, ulonglong code) +{ + uint chr; + if (!element->a.leaf.null) + { + chr=element->a.leaf.element_nr; + huff_tree->code_len[chr]= (uchar) (8 * sizeof(ulonglong) - size); + huff_tree->code[chr]= (size == 8 * sizeof(ulonglong)) ? 0 : (code >> size); + if (huff_tree->height < 8 * sizeof(ulonglong) - size) + huff_tree->height= 8 * sizeof(ulonglong) - size; + } + else + { + size--; + make_traverse_code_tree(huff_tree,element->a.nod.left,size,code); + make_traverse_code_tree(huff_tree, element->a.nod.right, size, + code + (((ulonglong) 1) << size)); + } + return; +} + + +/* + Convert a value into binary digits. + + SYNOPSIS + bindigits() + value The value. + length The number of low order bits to convert. + + NOTE + The result string is in static storage. It is reused on every call. + So you cannot use it twice in one expression. + + RETURN + A pointer to a static NUL-terminated string. + */ + +static char *bindigits(ulonglong value, uint bits) +{ + static char digits[72]; + char *ptr= digits; + uint idx= bits; + + DBUG_ASSERT(idx < sizeof(digits)); + while (idx) + *(ptr++)= '0' + ((char) (value >> (--idx)) & (char) 1); + *ptr= '\0'; + return digits; +} + + +/* + Convert a value into hexadecimal digits. + + SYNOPSIS + hexdigits() + value The value. + + NOTE + The result string is in static storage. It is reused on every call. + So you cannot use it twice in one expression. + + RETURN + A pointer to a static NUL-terminated string. + */ + +static char *hexdigits(ulonglong value) +{ + static char digits[20]; + char *ptr= digits; + uint idx= 2 * sizeof(value); /* Two hex digits per byte. */ + + DBUG_ASSERT(idx < sizeof(digits)); + while (idx) + { + if ((*(ptr++)= '0' + ((char) (value >> (4 * (--idx))) & (char) 0xf)) > '9') + *(ptr - 1)+= 'a' - '9' - 1; + } + *ptr= '\0'; + return digits; +} + + + /* Write header to new packed data file */ + +static int write_header(PACK_MRG_INFO *mrg,uint head_length,uint trees, + my_off_t tot_elements,my_off_t filelength) +{ + uchar *buff= (uchar*) file_buffer.pos; + + bzero(buff,HEAD_LENGTH); + memcpy(buff,myisam_pack_file_magic,4); + int4store(buff+4,head_length); + int4store(buff+8, mrg->min_pack_length); + int4store(buff+12,mrg->max_pack_length); + int4store(buff+16,tot_elements); + int4store(buff+20,intervall_length); + int2store(buff+24,trees); + buff[26]=(char) mrg->ref_length; + /* Save record pointer length */ + buff[27]= (uchar) mi_get_pointer_length((ulonglong) filelength,2); + if (test_only) + return 0; + my_seek(file_buffer.file,0L,MY_SEEK_SET,MYF(0)); + return my_write(file_buffer.file,(const uchar *) file_buffer.pos,HEAD_LENGTH, + MYF(MY_WME | MY_NABP | MY_WAIT_IF_FULL)) != 0; +} + + /* Write fieldinfo to new packed file */ + +static void write_field_info(HUFF_COUNTS *counts, uint fields, uint trees) +{ + reg1 uint i; + uint huff_tree_bits; + huff_tree_bits=max_bit(trees ? trees-1 : 0); + + DBUG_PRINT("info", (" ")); + DBUG_PRINT("info", ("column types:")); + DBUG_PRINT("info", ("FIELD_NORMAL 0")); + DBUG_PRINT("info", ("FIELD_SKIP_ENDSPACE 1")); + DBUG_PRINT("info", ("FIELD_SKIP_PRESPACE 2")); + DBUG_PRINT("info", ("FIELD_SKIP_ZERO 3")); + DBUG_PRINT("info", ("FIELD_BLOB 4")); + DBUG_PRINT("info", ("FIELD_CONSTANT 5")); + DBUG_PRINT("info", ("FIELD_INTERVALL 6")); + DBUG_PRINT("info", ("FIELD_ZERO 7")); + DBUG_PRINT("info", ("FIELD_VARCHAR 8")); + DBUG_PRINT("info", ("FIELD_CHECK 9")); + DBUG_PRINT("info", (" ")); + DBUG_PRINT("info", ("pack type as a set of flags:")); + DBUG_PRINT("info", ("PACK_TYPE_SELECTED 1")); + DBUG_PRINT("info", ("PACK_TYPE_SPACE_FIELDS 2")); + DBUG_PRINT("info", ("PACK_TYPE_ZERO_FILL 4")); + DBUG_PRINT("info", (" ")); + if (verbose >= 2) + { + printf("\n"); + printf("column types:\n"); + printf("FIELD_NORMAL 0\n"); + printf("FIELD_SKIP_ENDSPACE 1\n"); + printf("FIELD_SKIP_PRESPACE 2\n"); + printf("FIELD_SKIP_ZERO 3\n"); + printf("FIELD_BLOB 4\n"); + printf("FIELD_CONSTANT 5\n"); + printf("FIELD_INTERVALL 6\n"); + printf("FIELD_ZERO 7\n"); + printf("FIELD_VARCHAR 8\n"); + printf("FIELD_CHECK 9\n"); + printf("\n"); + printf("pack type as a set of flags:\n"); + printf("PACK_TYPE_SELECTED 1\n"); + printf("PACK_TYPE_SPACE_FIELDS 2\n"); + printf("PACK_TYPE_ZERO_FILL 4\n"); + printf("\n"); + } + for (i=0 ; i++ < fields ; counts++) + { + write_bits((ulonglong) (int) counts->field_type, 5); + write_bits(counts->pack_type,6); + if (counts->pack_type & PACK_TYPE_ZERO_FILL) + write_bits(counts->max_zero_fill,5); + else + write_bits(counts->length_bits,5); + write_bits((ulonglong) counts->tree->tree_number - 1, huff_tree_bits); + DBUG_PRINT("info", ("column: %3u type: %2u pack: %2u zero: %4u " + "lbits: %2u tree: %2u length: %4u", + i , counts->field_type, counts->pack_type, + counts->max_zero_fill, counts->length_bits, + counts->tree->tree_number, counts->field_length)); + if (verbose >= 2) + printf("column: %3u type: %2u pack: %2u zero: %4u lbits: %2u " + "tree: %2u length: %4u\n", i , counts->field_type, + counts->pack_type, counts->max_zero_fill, counts->length_bits, + counts->tree->tree_number, counts->field_length); + } + flush_bits(); + return; +} + + /* Write all huff_trees to new datafile. Return tot count of + elements in all trees + Returns 0 on error */ + +static my_off_t write_huff_tree(HUFF_TREE *huff_tree, uint trees) +{ + uint i,int_length; + uint tree_no; + uint codes; + uint errors= 0; + uint *packed_tree,*offset,length; + my_off_t elements; + + /* Find the highest number of elements in the trees. */ + for (i=length=0 ; i < trees ; i++) + if (huff_tree[i].tree_number > 0 && huff_tree[i].elements > length) + length=huff_tree[i].elements; + /* + Allocate a buffer for packing a decode tree. Two numbers per element + (left child and right child). + */ + if (!(packed_tree=(uint*) my_alloca(sizeof(uint)*length*2))) + { + my_error(EE_OUTOFMEMORY, MYF(ME_BELL+ME_FATAL), + sizeof(uint)*length*2); + return 0; + } + + DBUG_PRINT("info", (" ")); + if (verbose >= 2) + printf("\n"); + tree_no= 0; + intervall_length=0; + for (elements=0; trees-- ; huff_tree++) + { + /* Skip columns that have been joined with other columns. */ + if (huff_tree->tree_number == 0) + continue; /* Deleted tree */ + tree_no++; + DBUG_PRINT("info", (" ")); + if (verbose >= 3) + printf("\n"); + /* Count the total number of elements (byte codes or column values). */ + elements+=huff_tree->elements; + huff_tree->max_offset=2; + /* Build a tree of offsets and codes for decoding in 'packed_tree'. */ + if (huff_tree->elements <= 1) + offset=packed_tree; + else + offset=make_offset_code_tree(huff_tree,huff_tree->root,packed_tree); + + /* This should be the same as 'length' above. */ + huff_tree->offset_bits=max_bit(huff_tree->max_offset); + + /* + Since we check this during collecting the distinct column values, + this should never happen. + */ + if (huff_tree->max_offset >= IS_OFFSET) + { /* This should be impossible */ + (void) fprintf(stderr, "Tree offset got too big: %d, aborted\n", + huff_tree->max_offset); + my_afree((uchar*) packed_tree); + return 0; + } + + DBUG_PRINT("info", ("pos: %lu elements: %u tree-elements: %lu " + "char_bits: %u\n", + (ulong) (file_buffer.pos - file_buffer.buffer), + huff_tree->elements, (ulong) (offset - packed_tree), + huff_tree->char_bits)); + if (!huff_tree->counts->tree_buff) + { + /* We do a byte compression on this column. Mark with bit 0. */ + write_bits(0,1); + write_bits(huff_tree->min_chr,8); + write_bits(huff_tree->elements,9); + write_bits(huff_tree->char_bits,5); + write_bits(huff_tree->offset_bits,5); + int_length=0; + } + else + { + int_length=(uint) (huff_tree->counts->tree_pos - + huff_tree->counts->tree_buff); + /* We have distinct column values for this column. Mark with bit 1. */ + write_bits(1,1); + write_bits(huff_tree->elements,15); + write_bits(int_length,16); + write_bits(huff_tree->char_bits,5); + write_bits(huff_tree->offset_bits,5); + intervall_length+=int_length; + } + DBUG_PRINT("info", ("tree: %2u elements: %4u char_bits: %2u " + "offset_bits: %2u %s: %5u codelen: %2u", + tree_no, huff_tree->elements, huff_tree->char_bits, + huff_tree->offset_bits, huff_tree->counts->tree_buff ? + "bufflen" : "min_chr", huff_tree->counts->tree_buff ? + int_length : huff_tree->min_chr, huff_tree->height)); + if (verbose >= 2) + printf("tree: %2u elements: %4u char_bits: %2u offset_bits: %2u " + "%s: %5u codelen: %2u\n", tree_no, huff_tree->elements, + huff_tree->char_bits, huff_tree->offset_bits, + huff_tree->counts->tree_buff ? "bufflen" : "min_chr", + huff_tree->counts->tree_buff ? int_length : + huff_tree->min_chr, huff_tree->height); + + /* Check that the code tree length matches the element count. */ + length=(uint) (offset-packed_tree); + if (length != huff_tree->elements*2-2) + { + (void) fprintf(stderr, "error: Huff-tree-length: %d != calc_length: %d\n", + length, huff_tree->elements * 2 - 2); + errors++; + break; + } + + for (i=0 ; i < length ; i++) + { + if (packed_tree[i] & IS_OFFSET) + write_bits(packed_tree[i] - IS_OFFSET+ (1 << huff_tree->offset_bits), + huff_tree->offset_bits+1); + else + write_bits(packed_tree[i]-huff_tree->min_chr,huff_tree->char_bits+1); + DBUG_PRINT("info", ("tree[0x%04x]: %s0x%04x", + i, (packed_tree[i] & IS_OFFSET) ? + " -> " : "", (packed_tree[i] & IS_OFFSET) ? + packed_tree[i] - IS_OFFSET + i : packed_tree[i])); + if (verbose >= 3) + printf("tree[0x%04x]: %s0x%04x\n", + i, (packed_tree[i] & IS_OFFSET) ? " -> " : "", + (packed_tree[i] & IS_OFFSET) ? + packed_tree[i] - IS_OFFSET + i : packed_tree[i]); + } + flush_bits(); + + /* + Display coding tables and check their correctness. + */ + codes= huff_tree->counts->tree_buff ? huff_tree->elements : 256; + for (i= 0; i < codes; i++) + { + ulonglong code; + uint bits; + uint len; + uint idx; + + if (! (len= huff_tree->code_len[i])) + continue; + DBUG_PRINT("info", ("code[0x%04x]: 0x%s bits: %2u bin: %s", i, + hexdigits(huff_tree->code[i]), huff_tree->code_len[i], + bindigits(huff_tree->code[i], + huff_tree->code_len[i]))); + if (verbose >= 3) + printf("code[0x%04x]: 0x%s bits: %2u bin: %s\n", i, + hexdigits(huff_tree->code[i]), huff_tree->code_len[i], + bindigits(huff_tree->code[i], huff_tree->code_len[i])); + + /* Check that the encode table decodes correctly. */ + code= 0; + bits= 0; + idx= 0; + DBUG_EXECUTE_IF("forcechkerr1", len--;); + DBUG_EXECUTE_IF("forcechkerr2", bits= 8 * sizeof(code);); + DBUG_EXECUTE_IF("forcechkerr3", idx= length;); + for (;;) + { + if (! len) + { + (void) fflush(stdout); + (void) fprintf(stderr, "error: code 0x%s with %u bits not found\n", + hexdigits(huff_tree->code[i]), huff_tree->code_len[i]); + errors++; + break; + } + code<<= 1; + code|= (huff_tree->code[i] >> (--len)) & 1; + bits++; + if (bits > 8 * sizeof(code)) + { + (void) fflush(stdout); + (void) fprintf(stderr, "error: Huffman code too long: %u/%u\n", + bits, (uint) (8 * sizeof(code))); + errors++; + break; + } + idx+= (uint) code & 1; + if (idx >= length) + { + (void) fflush(stdout); + (void) fprintf(stderr, "error: illegal tree offset: %u/%u\n", + idx, length); + errors++; + break; + } + if (packed_tree[idx] & IS_OFFSET) + idx+= packed_tree[idx] & ~IS_OFFSET; + else + break; /* Hit a leaf. This contains the result value. */ + } + if (errors) + break; + + DBUG_EXECUTE_IF("forcechkerr4", packed_tree[idx]++;); + if (packed_tree[idx] != i) + { + (void) fflush(stdout); + (void) fprintf(stderr, "error: decoded value 0x%04x should be: 0x%04x\n", + packed_tree[idx], i); + errors++; + break; + } + } /*end for (codes)*/ + if (errors) + break; + + /* Write column values in case of distinct column value compression. */ + if (huff_tree->counts->tree_buff) + { + for (i=0 ; i < int_length ; i++) + { + write_bits((ulonglong) (uchar) huff_tree->counts->tree_buff[i], 8); + DBUG_PRINT("info", ("column_values[0x%04x]: 0x%02x", + i, (uchar) huff_tree->counts->tree_buff[i])); + if (verbose >= 3) + printf("column_values[0x%04x]: 0x%02x\n", + i, (uchar) huff_tree->counts->tree_buff[i]); + } + } + flush_bits(); + } + DBUG_PRINT("info", (" ")); + if (verbose >= 2) + printf("\n"); + my_afree((uchar*) packed_tree); + if (errors) + { + (void) fprintf(stderr, "Error: Generated decode trees are corrupt. Stop.\n"); + return 0; + } + return elements; +} + + +static uint *make_offset_code_tree(HUFF_TREE *huff_tree, HUFF_ELEMENT *element, + uint *offset) +{ + uint *prev_offset; + + prev_offset= offset; + /* + 'a.leaf.null' takes the same place as 'a.nod.left'. If this is null, + then there is no left child and, hence no right child either. This + is a property of a binary tree. An element is either a node with two + childs, or a leaf without childs. + + The current element is always a node with two childs. Go left first. + */ + if (!element->a.nod.left->a.leaf.null) + { + /* Store the byte code or the index of the column value. */ + prev_offset[0] =(uint) element->a.nod.left->a.leaf.element_nr; + offset+=2; + } + else + { + /* + Recursively traverse the tree to the left. Mark it as an offset to + another tree node (in contrast to a byte code or column value index). + */ + prev_offset[0]= IS_OFFSET+2; + offset=make_offset_code_tree(huff_tree,element->a.nod.left,offset+2); + } + + /* Now, check the right child. */ + if (!element->a.nod.right->a.leaf.null) + { + /* Store the byte code or the index of the column value. */ + prev_offset[1]=element->a.nod.right->a.leaf.element_nr; + return offset; + } + else + { + /* + Recursively traverse the tree to the right. Mark it as an offset to + another tree node (in contrast to a byte code or column value index). + */ + uint temp=(uint) (offset-prev_offset-1); + prev_offset[1]= IS_OFFSET+ temp; + if (huff_tree->max_offset < temp) + huff_tree->max_offset = temp; + return make_offset_code_tree(huff_tree,element->a.nod.right,offset); + } +} + + /* Get number of bits neaded to represent value */ + +static uint max_bit(register uint value) +{ + reg2 uint power=1; + + while ((value>>=1)) + power++; + return (power); +} + + +static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) +{ + int error; + uint i,max_calc_length,pack_ref_length,min_record_length,max_record_length, + intervall,field_length,max_pack_length,pack_blob_length; + my_off_t record_count; + char llbuf[32]; + ulong length,pack_length; + uchar *record,*pos,*end_pos,*record_pos,*start_pos; + HUFF_COUNTS *count,*end_count; + HUFF_TREE *tree; + MI_INFO *isam_file=mrg->file[0]; + uint pack_version= (uint) isam_file->s->pack.version; + DBUG_ENTER("compress_isam_file"); + + /* Allocate a buffer for the records (excluding blobs). */ + if (!(record=(uchar*) my_alloca(isam_file->s->base.reclength))) + return -1; + + end_count=huff_counts+isam_file->s->base.fields; + min_record_length= (uint) ~0; + max_record_length=0; + + /* + Calculate the maximum number of bits required to pack the records. + Remember to understand 'max_zero_fill' as 'min_zero_fill'. + The tree height determines the maximum number of bits per value. + Some fields skip leading or trailing spaces or zeroes. The skipped + number of bytes is encoded by 'length_bits' bits. + Empty blobs and varchar are encoded with a single 1 bit. Other blobs + and varchar get a leading 0 bit. + */ + for (i=max_calc_length=0 ; i < isam_file->s->base.fields ; i++) + { + if (!(huff_counts[i].pack_type & PACK_TYPE_ZERO_FILL)) + huff_counts[i].max_zero_fill=0; + if (huff_counts[i].field_type == FIELD_CONSTANT || + huff_counts[i].field_type == FIELD_ZERO || + huff_counts[i].field_type == FIELD_CHECK) + continue; + if (huff_counts[i].field_type == FIELD_INTERVALL) + max_calc_length+=huff_counts[i].tree->height; + else if (huff_counts[i].field_type == FIELD_BLOB || + huff_counts[i].field_type == FIELD_VARCHAR) + max_calc_length+=huff_counts[i].tree->height*huff_counts[i].max_length + huff_counts[i].length_bits +1; + else + max_calc_length+= + (huff_counts[i].field_length - huff_counts[i].max_zero_fill)* + huff_counts[i].tree->height+huff_counts[i].length_bits; + } + max_calc_length= (max_calc_length + 7) / 8; + pack_ref_length= calc_pack_length(pack_version, max_calc_length); + record_count=0; + /* 'max_blob_length' is the max length of all blobs of a record. */ + pack_blob_length= isam_file->s->base.blobs ? + calc_pack_length(pack_version, mrg->max_blob_length) : 0; + max_pack_length=pack_ref_length+pack_blob_length; + + DBUG_PRINT("fields", ("===")); + mrg_reset(mrg); + while ((error=mrg_rrnd(mrg,record)) != HA_ERR_END_OF_FILE) + { + ulong tot_blob_length=0; + if (! error) + { + if (flush_buffer((ulong) max_calc_length + (ulong) max_pack_length)) + break; + record_pos= (uchar*) file_buffer.pos; + file_buffer.pos+=max_pack_length; + for (start_pos=record, count= huff_counts; count < end_count ; count++) + { + end_pos=start_pos+(field_length=count->field_length); + tree=count->tree; + + DBUG_PRINT("fields", ("column: %3lu type: %2u pack: %2u zero: %4u " + "lbits: %2u tree: %2u length: %4u", + (ulong) (count - huff_counts + 1), + count->field_type, + count->pack_type, count->max_zero_fill, + count->length_bits, count->tree->tree_number, + count->field_length)); + + /* Check if the column contains spaces only. */ + if (count->pack_type & PACK_TYPE_SPACE_FIELDS) + { + for (pos=start_pos ; *pos == ' ' && pos < end_pos; pos++) ; + if (pos == end_pos) + { + DBUG_PRINT("fields", + ("PACK_TYPE_SPACE_FIELDS spaces only, bits: 1")); + DBUG_PRINT("fields", ("---")); + write_bits(1,1); + start_pos=end_pos; + continue; + } + DBUG_PRINT("fields", + ("PACK_TYPE_SPACE_FIELDS not only spaces, bits: 1")); + write_bits(0,1); + } + end_pos-=count->max_zero_fill; + field_length-=count->max_zero_fill; + + switch (count->field_type) { + case FIELD_SKIP_ZERO: + if (!memcmp((uchar*) start_pos,zero_string,field_length)) + { + DBUG_PRINT("fields", ("FIELD_SKIP_ZERO zeroes only, bits: 1")); + write_bits(1,1); + start_pos=end_pos; + break; + } + DBUG_PRINT("fields", ("FIELD_SKIP_ZERO not only zeroes, bits: 1")); + write_bits(0,1); + /* Fall through */ + case FIELD_NORMAL: + DBUG_PRINT("fields", ("FIELD_NORMAL %lu bytes", + (ulong) (end_pos - start_pos))); + for ( ; start_pos < end_pos ; start_pos++) + { + DBUG_PRINT("fields", + ("value: 0x%02x code: 0x%s bits: %2u bin: %s", + (uchar) *start_pos, + hexdigits(tree->code[(uchar) *start_pos]), + (uint) tree->code_len[(uchar) *start_pos], + bindigits(tree->code[(uchar) *start_pos], + (uint) tree->code_len[(uchar) *start_pos]))); + write_bits(tree->code[(uchar) *start_pos], + (uint) tree->code_len[(uchar) *start_pos]); + } + break; + case FIELD_SKIP_ENDSPACE: + for (pos=end_pos ; pos > start_pos && pos[-1] == ' ' ; pos--) ; + length= (ulong) (end_pos - pos); + if (count->pack_type & PACK_TYPE_SELECTED) + { + if (length > count->min_space) + { + DBUG_PRINT("fields", + ("FIELD_SKIP_ENDSPACE more than min_space, bits: 1")); + DBUG_PRINT("fields", + ("FIELD_SKIP_ENDSPACE skip %lu/%u bytes, bits: %2u", + length, field_length, count->length_bits)); + write_bits(1,1); + write_bits(length,count->length_bits); + } + else + { + DBUG_PRINT("fields", + ("FIELD_SKIP_ENDSPACE not more than min_space, " + "bits: 1")); + write_bits(0,1); + pos=end_pos; + } + } + else + { + DBUG_PRINT("fields", + ("FIELD_SKIP_ENDSPACE skip %lu/%u bytes, bits: %2u", + length, field_length, count->length_bits)); + write_bits(length,count->length_bits); + } + /* Encode all significant bytes. */ + DBUG_PRINT("fields", ("FIELD_SKIP_ENDSPACE %lu bytes", + (ulong) (pos - start_pos))); + for ( ; start_pos < pos ; start_pos++) + { + DBUG_PRINT("fields", + ("value: 0x%02x code: 0x%s bits: %2u bin: %s", + (uchar) *start_pos, + hexdigits(tree->code[(uchar) *start_pos]), + (uint) tree->code_len[(uchar) *start_pos], + bindigits(tree->code[(uchar) *start_pos], + (uint) tree->code_len[(uchar) *start_pos]))); + write_bits(tree->code[(uchar) *start_pos], + (uint) tree->code_len[(uchar) *start_pos]); + } + start_pos=end_pos; + break; + case FIELD_SKIP_PRESPACE: + for (pos=start_pos ; pos < end_pos && pos[0] == ' ' ; pos++) ; + length= (ulong) (pos - start_pos); + if (count->pack_type & PACK_TYPE_SELECTED) + { + if (length > count->min_space) + { + DBUG_PRINT("fields", + ("FIELD_SKIP_PRESPACE more than min_space, bits: 1")); + DBUG_PRINT("fields", + ("FIELD_SKIP_PRESPACE skip %lu/%u bytes, bits: %2u", + length, field_length, count->length_bits)); + write_bits(1,1); + write_bits(length,count->length_bits); + } + else + { + DBUG_PRINT("fields", + ("FIELD_SKIP_PRESPACE not more than min_space, " + "bits: 1")); + pos=start_pos; + write_bits(0,1); + } + } + else + { + DBUG_PRINT("fields", + ("FIELD_SKIP_PRESPACE skip %lu/%u bytes, bits: %2u", + length, field_length, count->length_bits)); + write_bits(length,count->length_bits); + } + /* Encode all significant bytes. */ + DBUG_PRINT("fields", ("FIELD_SKIP_PRESPACE %lu bytes", + (ulong) (end_pos - start_pos))); + for (start_pos=pos ; start_pos < end_pos ; start_pos++) + { + DBUG_PRINT("fields", + ("value: 0x%02x code: 0x%s bits: %2u bin: %s", + (uchar) *start_pos, + hexdigits(tree->code[(uchar) *start_pos]), + (uint) tree->code_len[(uchar) *start_pos], + bindigits(tree->code[(uchar) *start_pos], + (uint) tree->code_len[(uchar) *start_pos]))); + write_bits(tree->code[(uchar) *start_pos], + (uint) tree->code_len[(uchar) *start_pos]); + } + break; + case FIELD_CONSTANT: + case FIELD_ZERO: + case FIELD_CHECK: + DBUG_PRINT("fields", ("FIELD_CONSTANT/ZERO/CHECK")); + start_pos=end_pos; + break; + case FIELD_INTERVALL: + global_count=count; + pos=(uchar*) tree_search(&count->int_tree, start_pos, + count->int_tree.custom_arg); + intervall=(uint) (pos - count->tree_buff)/field_length; + DBUG_PRINT("fields", ("FIELD_INTERVALL")); + DBUG_PRINT("fields", ("index: %4u code: 0x%s bits: %2u", + intervall, hexdigits(tree->code[intervall]), + (uint) tree->code_len[intervall])); + write_bits(tree->code[intervall],(uint) tree->code_len[intervall]); + start_pos=end_pos; + break; + case FIELD_BLOB: + { + ulong blob_length=_mi_calc_blob_length(field_length- + portable_sizeof_char_ptr, + start_pos); + /* Empty blobs are encoded with a single 1 bit. */ + if (!blob_length) + { + DBUG_PRINT("fields", ("FIELD_BLOB empty, bits: 1")); + write_bits(1,1); + } + else + { + uchar *blob,*blob_end; + DBUG_PRINT("fields", ("FIELD_BLOB not empty, bits: 1")); + write_bits(0,1); + /* Write the blob length. */ + DBUG_PRINT("fields", ("FIELD_BLOB %lu bytes, bits: %2u", + blob_length, count->length_bits)); + write_bits(blob_length,count->length_bits); + memcpy(&blob, end_pos-portable_sizeof_char_ptr, sizeof(char*)); + blob_end=blob+blob_length; + /* Encode the blob bytes. */ + for ( ; blob < blob_end ; blob++) + { + DBUG_PRINT("fields", + ("value: 0x%02x code: 0x%s bits: %2u bin: %s", + (uchar) *blob, hexdigits(tree->code[(uchar) *blob]), + (uint) tree->code_len[(uchar) *blob], + bindigits(tree->code[(uchar) *start_pos], + (uint)tree->code_len[(uchar) *start_pos]))); + write_bits(tree->code[(uchar) *blob], + (uint) tree->code_len[(uchar) *blob]); + } + tot_blob_length+=blob_length; + } + start_pos= end_pos; + break; + } + case FIELD_VARCHAR: + { + uint var_pack_length= HA_VARCHAR_PACKLENGTH(count->field_length-1); + ulong col_length= (var_pack_length == 1 ? + (uint) *(uchar*) start_pos : + uint2korr(start_pos)); + /* Empty varchar are encoded with a single 1 bit. */ + if (!col_length) + { + DBUG_PRINT("fields", ("FIELD_VARCHAR empty, bits: 1")); + write_bits(1,1); /* Empty varchar */ + } + else + { + uchar *end= start_pos + var_pack_length + col_length; + DBUG_PRINT("fields", ("FIELD_VARCHAR not empty, bits: 1")); + write_bits(0,1); + /* Write the varchar length. */ + DBUG_PRINT("fields", ("FIELD_VARCHAR %lu bytes, bits: %2u", + col_length, count->length_bits)); + write_bits(col_length,count->length_bits); + /* Encode the varchar bytes. */ + for (start_pos+= var_pack_length ; start_pos < end ; start_pos++) + { + DBUG_PRINT("fields", + ("value: 0x%02x code: 0x%s bits: %2u bin: %s", + (uchar) *start_pos, + hexdigits(tree->code[(uchar) *start_pos]), + (uint) tree->code_len[(uchar) *start_pos], + bindigits(tree->code[(uchar) *start_pos], + (uint)tree->code_len[(uchar) *start_pos]))); + write_bits(tree->code[(uchar) *start_pos], + (uint) tree->code_len[(uchar) *start_pos]); + } + } + start_pos= end_pos; + break; + } + case FIELD_LAST: + case FIELD_enum_val_count: + abort(); /* Impossible */ + } + start_pos+=count->max_zero_fill; + DBUG_PRINT("fields", ("---")); + } + flush_bits(); + length=(ulong) ((uchar*) file_buffer.pos - record_pos) - max_pack_length; + pack_length= save_pack_length(pack_version, record_pos, length); + if (pack_blob_length) + pack_length+= save_pack_length(pack_version, record_pos + pack_length, + tot_blob_length); + DBUG_PRINT("fields", ("record: %lu length: %lu blob-length: %lu " + "length-bytes: %lu", (ulong) record_count, length, + tot_blob_length, pack_length)); + DBUG_PRINT("fields", ("===")); + + /* Correct file buffer if the header was smaller */ + if (pack_length != max_pack_length) + { + bmove(record_pos+pack_length,record_pos+max_pack_length,length); + file_buffer.pos-= (max_pack_length-pack_length); + } + if (length < (ulong) min_record_length) + min_record_length=(uint) length; + if (length > (ulong) max_record_length) + max_record_length=(uint) length; + record_count++; + if (write_loop && record_count % WRITE_COUNT == 0) + { + printf("%lu\r", (ulong) record_count); + (void) fflush(stdout); + } + } + else if (error != HA_ERR_RECORD_DELETED) + break; + } + if (error == HA_ERR_END_OF_FILE) + error=0; + else + { + (void) fprintf(stderr, "%s: Got error %d reading records\n", + my_progname, error); + } + if (verbose >= 2) + printf("wrote %s records.\n", llstr((longlong) record_count, llbuf)); + + my_afree((uchar*) record); + mrg->ref_length=max_pack_length; + mrg->min_pack_length=max_record_length ? min_record_length : 0; + mrg->max_pack_length=max_record_length; + DBUG_RETURN(error || error_on_write || flush_buffer(~(ulong) 0)); +} + + +static char *make_new_name(char *new_name, char *old_name) +{ + return fn_format(new_name,old_name,"",DATA_TMP_EXT,2+4); +} + +static char *make_old_name(char *new_name, char *old_name) +{ + return fn_format(new_name,old_name,"",OLD_EXT,2+4); +} + + /* rutines for bit writing buffer */ + +static void init_file_buffer(File file, pbool read_buffer) +{ + file_buffer.file=file; + file_buffer.buffer= (uchar*) my_malloc(PSI_NOT_INSTRUMENTED, ALIGN_SIZE(RECORD_CACHE_SIZE), + MYF(MY_WME)); + file_buffer.end=file_buffer.buffer+ALIGN_SIZE(RECORD_CACHE_SIZE)-8; + file_buffer.pos_in_file=0; + error_on_write=0; + if (read_buffer) + { + + file_buffer.pos=file_buffer.end; + file_buffer.bits=0; + } + else + { + file_buffer.pos=file_buffer.buffer; + file_buffer.bits=BITS_SAVED; + } + file_buffer.bitbucket= 0; +} + + +static int flush_buffer(ulong neaded_length) +{ + ulong length; + + /* + file_buffer.end is 8 bytes lower than the real end of the buffer. + This is done so that the end-of-buffer condition does not need to be + checked for every byte (see write_bits()). Consequently, + file_buffer.pos can become greater than file_buffer.end. The + algorithms in the other functions ensure that there will never be + more than 8 bytes written to the buffer without an end-of-buffer + check. So the buffer cannot be overrun. But we need to check for the + near-to-buffer-end condition to avoid a negative result, which is + casted to unsigned and thus becomes giant. + */ + if ((file_buffer.pos < file_buffer.end) && + ((ulong) (file_buffer.end - file_buffer.pos) > neaded_length)) + return 0; + length=(ulong) (file_buffer.pos-file_buffer.buffer); + file_buffer.pos=file_buffer.buffer; + file_buffer.pos_in_file+=length; + if (test_only) + return 0; + if (error_on_write|| my_write(file_buffer.file, + (const uchar*) file_buffer.buffer, + length, + MYF(MY_WME | MY_NABP | MY_WAIT_IF_FULL))) + { + error_on_write=1; + return 1; + } + + if (neaded_length != ~(ulong) 0 && + (ulong) (file_buffer.end-file_buffer.buffer) < neaded_length) + { + char *tmp; + neaded_length+=256; /* some margin */ + tmp= my_realloc(PSI_NOT_INSTRUMENTED, (char*) file_buffer.buffer, neaded_length,MYF(MY_WME)); + if (!tmp) + return 1; + file_buffer.pos= ((uchar*) tmp + + (ulong) (file_buffer.pos - file_buffer.buffer)); + file_buffer.buffer= (uchar*) tmp; + file_buffer.end= (uchar*) (tmp+neaded_length-8); + } + return 0; +} + + +static void end_file_buffer(void) +{ + my_free(file_buffer.buffer); +} + + /* output `bits` low bits of `value' */ + +static void write_bits(register ulonglong value, register uint bits) +{ + DBUG_ASSERT(((bits < 8 * sizeof(value)) && ! (value >> bits)) || + (bits == 8 * sizeof(value))); + + if ((file_buffer.bits-= (int) bits) >= 0) + { + file_buffer.bitbucket|= value << file_buffer.bits; + } + else + { + reg3 ulonglong bit_buffer; + bits= (uint) -file_buffer.bits; + bit_buffer= (file_buffer.bitbucket | + ((bits != 8 * sizeof(value)) ? (value >> bits) : 0)); +#if BITS_SAVED == 64 + *file_buffer.pos++= (uchar) (bit_buffer >> 56); + *file_buffer.pos++= (uchar) (bit_buffer >> 48); + *file_buffer.pos++= (uchar) (bit_buffer >> 40); + *file_buffer.pos++= (uchar) (bit_buffer >> 32); +#endif + *file_buffer.pos++= (uchar) (bit_buffer >> 24); + *file_buffer.pos++= (uchar) (bit_buffer >> 16); + *file_buffer.pos++= (uchar) (bit_buffer >> 8); + *file_buffer.pos++= (uchar) (bit_buffer); + + if (bits != 8 * sizeof(value)) + value&= (((ulonglong) 1) << bits) - 1; + if (file_buffer.pos >= file_buffer.end) + (void) flush_buffer(~ (ulong) 0); + file_buffer.bits=(int) (BITS_SAVED - bits); + file_buffer.bitbucket= value << (BITS_SAVED - bits); + } + return; +} + + /* Flush bits in bit_buffer to buffer */ + +static void flush_bits(void) +{ + int bits; + ulonglong bit_buffer; + + bits= file_buffer.bits & ~7; + if (bits != BITS_SAVED) + { + bit_buffer= file_buffer.bitbucket >> bits; + bits= BITS_SAVED - bits; + while (bits > 0) + { + bits-= 8; + *file_buffer.pos++= (uchar) (bit_buffer >> bits); + } + } + if (file_buffer.pos >= file_buffer.end) + (void) flush_buffer(~ (ulong) 0); + file_buffer.bits= BITS_SAVED; + file_buffer.bitbucket= 0; +} + + +/**************************************************************************** +** functions to handle the joined files +****************************************************************************/ + +static int save_state(MI_INFO *isam_file,PACK_MRG_INFO *mrg,my_off_t new_length, + ha_checksum crc) +{ + MYISAM_SHARE *share=isam_file->s; + uint options=mi_uint2korr(share->state.header.options); + uint key; + DBUG_ENTER("save_state"); + + options|= (HA_OPTION_COMPRESS_RECORD | HA_OPTION_READ_ONLY_DATA | + (share->options & HA_OPTION_NULL_FIELDS)); + mi_int2store(share->state.header.options,options); + + share->state.state.data_file_length=new_length; + share->state.state.del=0; + share->state.state.empty=0; + share->state.dellink= HA_OFFSET_ERROR; + share->state.split=(ha_rows) mrg->records; + share->state.version=(ulong) time((time_t*) 0); + if (! mi_is_all_keys_active(share->state.key_map, share->base.keys)) + { + /* + Some indexes are disabled, cannot use current key_file_length value + as an estimate of upper bound of index file size. Use packed data file + size instead. + */ + share->state.state.key_file_length= new_length; + } + /* + If there are no disabled indexes, keep key_file_length value from + original file so "myisamchk -rq" can use this value (this is necessary + because index size cannot be easily calculated for fulltext keys) + */ + mi_clear_all_keys_active(share->state.key_map); + for (key=0 ; key < share->base.keys ; key++) + share->state.key_root[key]= HA_OFFSET_ERROR; + for (key=0 ; key < share->state.header.max_block_size_index ; key++) + share->state.key_del[key]= HA_OFFSET_ERROR; + isam_file->state->checksum=crc; /* Save crc here */ + share->changed=1; /* Force write of header */ + share->state.open_count=0; + share->global_changed=0; + (void) my_chsize(share->kfile, share->base.keystart, 0, MYF(0)); + if (share->base.keys) + isamchk_neaded=1; + DBUG_RETURN(mi_state_info_write(share->kfile,&share->state,1+2)); +} + + +static int save_state_mrg(File file,PACK_MRG_INFO *mrg,my_off_t new_length, + ha_checksum crc) +{ + MI_STATE_INFO state; + MI_INFO *isam_file=mrg->file[0]; + uint options; + DBUG_ENTER("save_state_mrg"); + + state= isam_file->s->state; + options= (mi_uint2korr(state.header.options) | HA_OPTION_COMPRESS_RECORD | + HA_OPTION_READ_ONLY_DATA | + (isam_file->s->options & HA_OPTION_NULL_FIELDS)); + mi_int2store(state.header.options,options); + state.state.data_file_length=new_length; + state.state.del=0; + state.state.empty=0; + state.state.records=state.split=(ha_rows) mrg->records; + /* See comment above in save_state about key_file_length handling. */ + if (mrg->src_file_has_indexes_disabled) + { + isam_file->s->state.state.key_file_length= + MY_MAX(isam_file->s->state.state.key_file_length, new_length); + } + state.dellink= HA_OFFSET_ERROR; + state.version=(ulong) time((time_t*) 0); + mi_clear_all_keys_active(state.key_map); + state.state.checksum=crc; + if (isam_file->s->base.keys) + isamchk_neaded=1; + state.changed=STATE_CHANGED | STATE_NOT_ANALYZED; /* Force check of table */ + DBUG_RETURN (mi_state_info_write(file,&state,1+2)); +} + + +/* reset for mrg_rrnd */ + +static void mrg_reset(PACK_MRG_INFO *mrg) +{ + if (mrg->current) + { + mi_extra(*mrg->current, HA_EXTRA_NO_CACHE, 0); + mrg->current=0; + } +} + +static int mrg_rrnd(PACK_MRG_INFO *info,uchar *buf) +{ + int error; + MI_INFO *isam_info; + my_off_t filepos; + + if (!info->current) + { + isam_info= *(info->current=info->file); + info->end=info->current+info->count; + mi_reset(isam_info); + mi_extra(isam_info, HA_EXTRA_CACHE, 0); + filepos=isam_info->s->pack.header_length; + } + else + { + isam_info= *info->current; + filepos= isam_info->nextpos; + } + + for (;;) + { + isam_info->update&= HA_STATE_CHANGED; + if (!(error=(*isam_info->s->read_rnd)(isam_info,(uchar*) buf, + filepos, 1)) || + error != HA_ERR_END_OF_FILE) + return (error); + mi_extra(isam_info,HA_EXTRA_NO_CACHE, 0); + if (info->current+1 == info->end) + return(HA_ERR_END_OF_FILE); + info->current++; + isam_info= *info->current; + filepos=isam_info->s->pack.header_length; + mi_reset(isam_info); + mi_extra(isam_info,HA_EXTRA_CACHE, 0); + } +} + + +static int mrg_close(PACK_MRG_INFO *mrg) +{ + uint i; + int error=0; + for (i=0 ; i < mrg->count ; i++) + error|=mi_close(mrg->file[i]); + if (mrg->free_file) + my_free(mrg->file); + return error; +} + + +#if !defined(DBUG_OFF) +/* + Fake the counts to get big Huffman codes. + + SYNOPSIS + fakebigcodes() + huff_counts A pointer to the counts array. + end_count A pointer past the counts array. + + DESCRIPTION + + Huffman coding works by removing the two least frequent values from + the list of values and add a new value with the sum of their + incidences in a loop until only one value is left. Every time a + value is reused for a new value, it gets one more bit for its + encoding. Hence, the least frequent values get the longest codes. + + To get a maximum code length for a value, two of the values must + have an incidence of 1. As their sum is 2, the next infrequent value + must have at least an incidence of 2, then 4, 8, 16 and so on. This + means that one needs 2**n bytes (values) for a code length of n + bits. However, using more distinct values forces the use of longer + codes, or reaching the code length with less total bytes (values). + + To get 64(32)-bit codes, I sort the counts by decreasing incidence. + I assign counts of 1 to the two most frequent values, a count of 2 + for the next one, then 4, 8, and so on until 2**64-1(2**30-1). All + the remaining values get 1. That way every possible byte has an + assigned code, though not all codes are used if not all byte values + are present in the column. + + This strategy would work with distinct column values too, but + requires that at least 64(32) values are present. To make things + easier here, I cancel all distinct column values and force byte + compression for all columns. + + RETURN + void +*/ + +static void fakebigcodes(HUFF_COUNTS *huff_counts, HUFF_COUNTS *end_count) +{ + HUFF_COUNTS *count; + my_off_t *cur_count_p; + my_off_t *end_count_p; + my_off_t **cur_sort_p; + my_off_t **end_sort_p; + my_off_t *sort_counts[256]; + my_off_t total; + DBUG_ENTER("fakebigcodes"); + + for (count= huff_counts; count < end_count; count++) + { + /* + Remove distinct column values. + */ + if (huff_counts->tree_buff) + { + my_free(huff_counts->tree_buff); + delete_tree(&huff_counts->int_tree, 0); + huff_counts->tree_buff= NULL; + DBUG_PRINT("fakebigcodes", ("freed distinct column values")); + } + + /* + Sort counts by decreasing incidence. + */ + cur_count_p= count->counts; + end_count_p= cur_count_p + 256; + cur_sort_p= sort_counts; + while (cur_count_p < end_count_p) + *(cur_sort_p++)= cur_count_p++; + (void) my_qsort(sort_counts, 256, sizeof(my_off_t*), (qsort_cmp) fakecmp); + + /* + Assign faked counts. + */ + cur_sort_p= sort_counts; +#if SIZEOF_LONG_LONG > 4 + end_sort_p= sort_counts + 8 * sizeof(ulonglong) - 1; +#else + end_sort_p= sort_counts + 8 * sizeof(ulonglong) - 2; +#endif + /* Most frequent value gets a faked count of 1. */ + **(cur_sort_p++)= 1; + total= 1; + while (cur_sort_p < end_sort_p) + { + **(cur_sort_p++)= total; + total<<= 1; + } + /* Set the last value. */ + **(cur_sort_p++)= --total; + /* + Set the remaining counts. + */ + end_sort_p= sort_counts + 256; + while (cur_sort_p < end_sort_p) + **(cur_sort_p++)= 1; + } + DBUG_VOID_RETURN; +} + + +/* + Compare two counts for reverse sorting. + + SYNOPSIS + fakecmp() + count1 One count. + count2 Another count. + + RETURN + 1 count1 < count2 + 0 count1 == count2 + -1 count1 > count2 +*/ + +static int fakecmp(my_off_t **count1, my_off_t **count2) +{ + return ((**count1 < **count2) ? 1 : + (**count1 > **count2) ? -1 : 0); +} +#endif + +#include "mi_extrafunc.h" diff --git a/storage/myisam/mysql-test/mtr2/README b/storage/myisam/mysql-test/mtr2/README new file mode 100644 index 00000000..5b2453d0 --- /dev/null +++ b/storage/myisam/mysql-test/mtr2/README @@ -0,0 +1,2 @@ +These tests don't have anything to do with the engine itself, +but they test how mysql-test handles overlays diff --git a/storage/myisam/mysql-test/mtr2/overlay.inc b/storage/myisam/mysql-test/mtr2/overlay.inc new file mode 100644 index 00000000..4113a306 --- /dev/null +++ b/storage/myisam/mysql-test/mtr2/overlay.inc @@ -0,0 +1,2 @@ +select 3; + diff --git a/storage/myisam/mysql-test/mtr2/single.rdiff b/storage/myisam/mysql-test/mtr2/single.rdiff new file mode 100644 index 00000000..fd590d53 --- /dev/null +++ b/storage/myisam/mysql-test/mtr2/single.rdiff @@ -0,0 +1,12 @@ +--- suite/mtr2/single.result 2013-11-10 03:58:37.000000000 +0400 ++++ suite/mtr2/single.reject 2013-11-10 03:59:08.000000000 +0400 +@@ -1,6 +1,6 @@ + select 1; + 1 + 1 +-select 2; +-2 +-2 ++select 3; ++3 ++3 diff --git a/storage/myisam/mysql-test/mtr2/suite.opt b/storage/myisam/mysql-test/mtr2/suite.opt new file mode 100644 index 00000000..8e7b7f9e --- /dev/null +++ b/storage/myisam/mysql-test/mtr2/suite.opt @@ -0,0 +1 @@ +--old diff --git a/storage/myisam/mysql-test/mtr2/suite.pm b/storage/myisam/mysql-test/mtr2/suite.pm new file mode 100644 index 00000000..0f287e4a --- /dev/null +++ b/storage/myisam/mysql-test/mtr2/suite.pm @@ -0,0 +1,9 @@ +package My::Suite::MTR2::MyISAM; + +@ISA = qw(My::Suite); + +sub skip_combinations {( + 'combinations' => [ '1st' ], +)} +bless { }; + diff --git a/storage/myisam/mysql-test/storage_engine/alter_table_online.rdiff b/storage/myisam/mysql-test/storage_engine/alter_table_online.rdiff new file mode 100644 index 00000000..58c7620f --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/alter_table_online.rdiff @@ -0,0 +1,35 @@ +--- suite/storage_engine/alter_table_online.result 2013-11-08 20:01:16.000000000 +0400 ++++ suite/storage_engine/alter_table_online.reject 2013-11-08 20:02:03.000000000 +0400 +@@ -23,12 +50,30 @@ + CREATE TABLE t1 (a <INT_COLUMN>, b <INT_COLUMN>, c <CHAR_COLUMN>) ENGINE=<STORAGE_ENGINE> <CUSTOM_TABLE_OPTIONS>; + INSERT INTO t1 (a,b,c) VALUES (1,100,'a'),(2,200,'b'),(3,300,'c'); + ALTER ONLINE TABLE t1 DROP COLUMN b, ADD b <INT_COLUMN>; ++ERROR 0A000: LOCK=NONE is not supported for this operation. Try LOCK=SHARED. ++# ERROR: Statement ended with errno 1845, errname ER_ALTER_OPERATION_NOT_SUPPORTED (expected to succeed) ++# ------------ UNEXPECTED RESULT ------------ ++# The statement|command finished with ER_ALTER_OPERATION_NOT_SUPPORTED. ++# Functionality or the mix could be unsupported|malfunctioning, or the problem was caused by previous errors. ++# You can change the engine code, or create an rdiff, or disable the test by adding it to disabled.def. ++# Further in this test, the message might sometimes be suppressed; a part of the test might be skipped. ++# Also, this problem may cause a chain effect (more errors of different kinds in the test). ++# ------------------------------------------- + ALTER ONLINE TABLE t1 MODIFY b BIGINT <CUSTOM_COL_OPTIONS>; +-ERROR 0A000: LOCK=NONE is not supported. Reason: Cannot change column type. Try LOCK=SHARED. ++ERROR 0A000: LOCK=NONE is not supported for this operation. Try LOCK=SHARED. ++# ERROR: Statement ended with errno 1845, errname ER_ALTER_OPERATION_NOT_SUPPORTED (expected results: ER_ALTER_OPERATION_NOT_SUPPORTED_REASON) + ALTER ONLINE TABLE t1 ENGINE=MEMORY; + ERROR 0A000: LOCK=NONE is not supported. Reason: COPY algorithm requires a lock. Try LOCK=SHARED. + DROP TABLE t1; + CREATE TABLE t1 (a <INT_COLUMN>, b <INT_COLUMN>, c <CHAR_COLUMN>) ENGINE=<STORAGE_ENGINE> <CUSTOM_TABLE_OPTIONS>; + ALTER ONLINE TABLE t1 ADD INDEX (b); +-ALTER ONLINE TABLE t1 DROP INDEX b; ++ERROR 0A000: LOCK=NONE is not supported for this operation. Try LOCK=SHARED. ++# ERROR: Statement ended with errno 1845, errname ER_ALTER_OPERATION_NOT_SUPPORTED (expected to succeed) ++# ------------ UNEXPECTED RESULT ------------ ++# The statement|command finished with ER_ALTER_OPERATION_NOT_SUPPORTED. ++# Adding an index or ALTER ONLINE or the mix could be unsupported|malfunctioning, or the problem was caused by previous errors. ++# You can change the engine code, or create an rdiff, or disable the test by adding it to disabled.def. ++# Further in this test, the message might sometimes be suppressed; a part of the test might be skipped. ++# Also, this problem may cause a chain effect (more errors of different kinds in the test). ++# ------------------------------------------- + DROP TABLE t1; diff --git a/storage/myisam/mysql-test/storage_engine/alter_tablespace.rdiff b/storage/myisam/mysql-test/storage_engine/alter_tablespace.rdiff new file mode 100644 index 00000000..a8c78b11 --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/alter_tablespace.rdiff @@ -0,0 +1,34 @@ +--- suite/storage_engine/alter_tablespace.result 2012-07-12 19:53:40.775419511 +0400 ++++ suite/storage_engine/alter_tablespace.reject 2012-07-15 16:21:14.910435703 +0400 +@@ -1,21 +1,14 @@ + DROP TABLE IF EXISTS t1, t2; + CREATE TABLE t1 (a <INT_COLUMN>) ENGINE=<STORAGE_ENGINE> <CUSTOM_TABLE_OPTIONS>; + ALTER TABLE t1 DISCARD TABLESPACE; +-DROP TABLE t1; +-CREATE TABLE t1 (a <INT_COLUMN>) ENGINE=<STORAGE_ENGINE> <CUSTOM_TABLE_OPTIONS>; +-INSERT INTO t1 (a) VALUES (1),(2); +-SELECT a FROM t1; +-a +-1 +-2 +-ALTER TABLE t1 DISCARD TABLESPACE; +-SELECT a FROM t1; +-ERROR HY000: Tablespace has been discarded for table `t1` +-ALTER TABLE t1 IMPORT TABLESPACE; +-Warnings: +-Warning 1810 IO Read error: (2, No such file or directory) Error opening './test/t1.cfg', will attempt to import without schema verification +-SELECT a FROM t1; +-a +-1 +-2 ++ERROR HY000: Storage engine <STORAGE_ENGINE> of the table `test`.`t1` doesn't have this option ++# ERROR: Statement ended with errno 1031, errname ER_ILLEGAL_HA (expected to succeed) ++# ------------ UNEXPECTED RESULT ------------ ++# [ ALTER TABLE t1 DISCARD TABLESPACE ] ++# The statement|command finished with ER_ILLEGAL_HA. ++# Tablespace operations or the syntax or the mix could be unsupported. ++# You can change the engine code, or create an rdiff, or disable the test by adding it to disabled.def. ++# Further in this test, the message might sometimes be suppressed; a part of the test might be skipped. ++# Also, this problem may cause a chain effect (more errors of different kinds in the test). ++# ------------------------------------------- + DROP TABLE t1; diff --git a/storage/myisam/mysql-test/storage_engine/check_table.rdiff b/storage/myisam/mysql-test/storage_engine/check_table.rdiff new file mode 100644 index 00000000..48e8fc2e --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/check_table.rdiff @@ -0,0 +1,20 @@ +--- suite/storage_engine/check_table.result 2012-07-15 04:19:07.782936394 +0400 ++++ suite/storage_engine/check_table.reject 2012-07-15 16:21:16.734412773 +0400 +@@ -18,7 +18,7 @@ + INSERT INTO t1 (a,b) VALUES (6,'f'); + CHECK TABLE t1 FAST; + Table Op Msg_type Msg_text +-test.t1 check status OK ++test.t1 check status Table is already up to date + INSERT INTO t1 (a,b) VALUES (7,'g'); + INSERT INTO t2 (a,b) VALUES (8,'h'); + CHECK TABLE t2, t1 MEDIUM; +@@ -52,7 +52,7 @@ + INSERT INTO t1 (a) VALUES (17),(120),(132); + CHECK TABLE t1 FAST; + Table Op Msg_type Msg_text +-test.t1 check status OK ++test.t1 check status Table is already up to date + INSERT INTO t1 (a) VALUES (801),(900),(7714); + CHECK TABLE t1 MEDIUM; + Table Op Msg_type Msg_text diff --git a/storage/myisam/mysql-test/storage_engine/define_engine.inc b/storage/myisam/mysql-test/storage_engine/define_engine.inc new file mode 100644 index 00000000..d5e74162 --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/define_engine.inc @@ -0,0 +1,45 @@ +########################################### +# +# This is a template of the include file define_engine.inc which +# should be placed in storage/<engine>/mysql-test/storage_engine folder. +# +################################ +# +# The name of the engine under test must be defined in $ENGINE variable. +# You can set it either here (uncomment and edit) or in your environment. +# +let $ENGINE = MyISAM; +# +################################ +# +# The following three variables define specific options for columns and tables. +# Normally there should be none needed, but for some engines it can be different. +# If the engine requires specific column option for all or indexed columns, +# set them inside the comment, e.g. /*!NOT NULL*/. +# Do the same for table options if needed, e.g. /*!INSERT_METHOD=LAST*/ + +let $default_col_opts = /*!*/; +let $default_col_indexed_opts = /*!*/; +let $default_tbl_opts = /*!*/; + +# INDEX, UNIQUE INDEX, PRIMARY KEY, special index type - choose the fist that the engine allows, +# or set it to /*!*/ if none is supported + +let $default_index = /*!INDEX*/; + +# If the engine does not support the following types, replace them with the closest possible + +let $default_int_type = INT(11); +let $default_char_type = CHAR(8); + +################################ + +--disable_query_log +--disable_result_log + +# Here you can place your custom MTR code which needs to be executed before each test, +# e.g. creation of an additional schema or table, etc. +# The cleanup part should be defined in cleanup_engine.inc + +--enable_query_log +--enable_result_log diff --git a/storage/myisam/mysql-test/storage_engine/foreign_keys.rdiff b/storage/myisam/mysql-test/storage_engine/foreign_keys.rdiff new file mode 100644 index 00000000..cf24f2c3 --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/foreign_keys.rdiff @@ -0,0 +1,145 @@ +--- suite/storage_engine/foreign_keys.result 2012-07-12 18:56:19.782678645 +0400 ++++ suite/storage_engine/foreign_keys.reject 2012-07-15 16:21:30.414240794 +0400 +@@ -12,29 +12,57 @@ + t2 CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` char(8) DEFAULT NULL, +- KEY `a` (`a`), +- CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`a`) REFERENCES `t1` (`a`) ++ KEY `a` (`a`) + ) ENGINE=<STORAGE_ENGINE> DEFAULT CHARSET=latin1 + INSERT INTO t2 (a,b) VALUES (1,'a'),(2,'b'); +-ERROR 23000: Cannot add or update a child row: a foreign key constraint fails (`test`.`t2`, CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`a`) REFERENCES `t1` (`a`)) ++# ERROR: Statement succeeded (expected results: ER_NO_REFERENCED_ROW_2) ++# ------------ UNEXPECTED RESULT ------------ ++# The statement|command succeeded unexpectedly. ++# Foreign keys or the mix could be unsupported|malfunctioning, or the problem was caused by previous errors. ++# You can change the engine code, or create an rdiff, or disable the test by adding it to disabled.def. ++# Further in this test, the message might sometimes be suppressed; a part of the test might be skipped. ++# Also, this problem may cause a chain effect (more errors of different kinds in the test). ++# ------------------------------------------- + INSERT INTO t1 (a,b) VALUES (1,'c'),(2,'d'); + INSERT INTO t2 (a,b) VALUES (1,'a'),(2,'b'); + UPDATE t2 SET a=a+1; +-ERROR 23000: Cannot add or update a child row: a foreign key constraint fails (`test`.`t2`, CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`a`) REFERENCES `t1` (`a`)) ++# ERROR: Statement succeeded (expected results: ER_NO_REFERENCED_ROW_2) ++# ------------ UNEXPECTED RESULT ------------ ++# The statement|command succeeded unexpectedly. ++# Foreign keys or the mix could be unsupported|malfunctioning, or the problem was caused by previous errors. ++# You can change the engine code, or create an rdiff, or disable the test by adding it to disabled.def. ++# Further in this test, the message might sometimes be suppressed; a part of the test might be skipped. ++# Also, this problem may cause a chain effect (more errors of different kinds in the test). ++# ------------------------------------------- + UPDATE t1 SET a=3 WHERE a=2; +-ERROR 23000: Cannot delete or update a parent row: a foreign key constraint fails (`test`.`t2`, CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`a`) REFERENCES `t1` (`a`)) ++# ERROR: Statement succeeded (expected results: ER_ROW_IS_REFERENCED_2) + DELETE FROM t1 WHERE a=2; +-ERROR 23000: Cannot delete or update a parent row: a foreign key constraint fails (`test`.`t2`, CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`a`) REFERENCES `t1` (`a`)) ++# ERROR: Statement succeeded (expected results: ER_ROW_IS_REFERENCED_2) ++# ------------ UNEXPECTED RESULT ------------ ++# The statement|command succeeded unexpectedly. ++# Foreign keys or the mix could be unsupported|malfunctioning, or the problem was caused by previous errors. ++# You can change the engine code, or create an rdiff, or disable the test by adding it to disabled.def. ++# Further in this test, the message might sometimes be suppressed; a part of the test might be skipped. ++# Also, this problem may cause a chain effect (more errors of different kinds in the test). ++# ------------------------------------------- + DELETE FROM t2 WHERE a=2; + SELECT a,b FROM t1; + a b + 1 c +-2 d ++3 d + SELECT a,b FROM t2; + a b +-1 a ++3 b ++3 b + DROP TABLE t1; +-ERROR 23000: Cannot delete or update a parent row: a foreign key constraint fails ++# ERROR: Statement succeeded (expected results: ER_ROW_IS_REFERENCED_2) ++# ------------ UNEXPECTED RESULT ------------ ++# The statement|command succeeded unexpectedly. ++# Foreign keys or the mix could be unsupported|malfunctioning, or the problem was caused by previous errors. ++# You can change the engine code, or create an rdiff, or disable the test by adding it to disabled.def. ++# Further in this test, the message might sometimes be suppressed; a part of the test might be skipped. ++# Also, this problem may cause a chain effect (more errors of different kinds in the test). ++# ------------------------------------------- + DROP TABLE t2; + CREATE TABLE t2 (a <INT_COLUMN>, + b <CHAR_COLUMN>, +@@ -46,26 +74,65 @@ + t2 CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` char(8) DEFAULT NULL, +- KEY `a` (`a`), +- CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`a`) REFERENCES `t1` (`a`) ON DELETE CASCADE ON UPDATE CASCADE ++ KEY `a` (`a`) + ) ENGINE=<STORAGE_ENGINE> DEFAULT CHARSET=latin1 + INSERT INTO t2 (a,b) VALUES (1,'a'),(2,'b'),(3,'c'),(4,'d'); +-ERROR 23000: Cannot add or update a child row: a foreign key constraint fails (`test`.`t2`, CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`a`) REFERENCES `t1` (`a`) ON DELETE CASCADE ON UPDATE CASCADE) ++# ERROR: Statement succeeded (expected results: ER_NO_REFERENCED_ROW_2) + INSERT INTO t1 (a,b) VALUES (3,'a'),(4,'a'); ++ERROR 42S02: Table 'test.t1' doesn't exist + INSERT INTO t2 (a,b) VALUES (1,'a'),(2,'b'),(3,'c'),(4,'d'),(4,'e'),(3,'a'); + UPDATE t1 SET a=a+1; ++ERROR 42S02: Table 'test.t1' doesn't exist ++# ------------ UNEXPECTED RESULT ------------ ++# The statement|command finished with ER_NO_SUCH_TABLE. ++# UPDATE or the mix could be unsupported|malfunctioning, or the problem was caused by previous errors. ++# You can change the engine code, or create an rdiff, or disable the test by adding it to disabled.def. ++# Further in this test, the message might sometimes be suppressed; a part of the test might be skipped. ++# Also, this problem may cause a chain effect (more errors of different kinds in the test). ++# ------------------------------------------- + SELECT a,b FROM t2; + a b +-5 a +-5 a +-5 b +-5 c +-5 d +-5 e ++1 a ++1 a ++2 b ++2 b ++3 a ++3 c ++3 c ++4 d ++4 d ++4 e + DELETE FROM t1 WHERE b='a' LIMIT 2; ++ERROR 42S02: Table 'test.t1' doesn't exist ++# ------------ UNEXPECTED RESULT ------------ ++# The statement|command finished with ER_NO_SUCH_TABLE. ++# DELETE or the mix could be unsupported|malfunctioning, or the problem was caused by previous errors. ++# You can change the engine code, or create an rdiff, or disable the test by adding it to disabled.def. ++# Further in this test, the message might sometimes be suppressed; a part of the test might be skipped. ++# Also, this problem may cause a chain effect (more errors of different kinds in the test). ++# ------------------------------------------- + SELECT a,b FROM t2; + a b ++1 a ++1 a ++2 b ++2 b ++3 a ++3 c ++3 c ++4 d ++4 d ++4 e + TRUNCATE TABLE t1; +-ERROR 42000: Cannot truncate a table referenced in a foreign key constraint (`test`.`t2`, CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`a`) REFERENCES `test`.`t1` (`a`)) ++ERROR 42S02: Table 'test.t1' doesn't exist ++# ERROR: Statement ended with errno 1146, errname ER_NO_SUCH_TABLE (expected results: ER_TRUNCATE_ILLEGAL_FK) ++# ------------ UNEXPECTED RESULT ------------ ++# The statement|command finished with ER_NO_SUCH_TABLE. ++# Foreign keys or the mix could be unsupported|malfunctioning, or the problem was caused by previous errors. ++# You can change the engine code, or create an rdiff, or disable the test by adding it to disabled.def. ++# Further in this test, the message might sometimes be suppressed; a part of the test might be skipped. ++# Also, this problem may cause a chain effect (more errors of different kinds in the test). ++# ------------------------------------------- + DROP TABLE t2; + DROP TABLE t1; ++ERROR 42S02: Unknown table 'test.t1' diff --git a/storage/myisam/mysql-test/storage_engine/index_type_hash.rdiff b/storage/myisam/mysql-test/storage_engine/index_type_hash.rdiff new file mode 100644 index 00000000..e7fa0013 --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/index_type_hash.rdiff @@ -0,0 +1,60 @@ +--- suite/storage_engine/index_type_hash.result 2012-07-15 01:10:17.919128889 +0400 ++++ suite/storage_engine/index_type_hash.reject 2012-07-15 16:21:32.806210722 +0400 +@@ -4,7 +4,7 @@ + ) ENGINE=<STORAGE_ENGINE> <CUSTOM_TABLE_OPTIONS>; + SHOW KEYS IN t1; + Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +-t1 1 a 1 a # # NULL NULL # HASH ++t1 1 a 1 a # # NULL NULL # BTREE + DROP TABLE t1; + CREATE TABLE t1 (a <INT_COLUMN>, + b <CHAR_COLUMN>, +@@ -12,8 +12,8 @@ + ) ENGINE=<STORAGE_ENGINE> <CUSTOM_TABLE_OPTIONS>; + SHOW KEYS IN t1; + Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +-t1 1 a_b 1 a # # NULL NULL # HASH a_b index +-t1 1 a_b 2 b # # NULL NULL # HASH a_b index ++t1 1 a_b 1 a # # NULL NULL # BTREE a_b index ++t1 1 a_b 2 b # # NULL NULL # BTREE a_b index + DROP TABLE t1; + CREATE TABLE t1 (a <INT_COLUMN>, + b <CHAR_COLUMN>, +@@ -22,8 +22,8 @@ + ) ENGINE=<STORAGE_ENGINE> <CUSTOM_TABLE_OPTIONS>; + SHOW KEYS IN t1; + Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +-t1 1 a 1 a # # NULL NULL # HASH +-t1 1 b 1 b # # NULL NULL # HASH ++t1 1 a 1 a # # NULL NULL # BTREE ++t1 1 b 1 b # # NULL NULL # BTREE + DROP TABLE t1; + CREATE TABLE t1 (a <INT_COLUMN>, + b <CHAR_COLUMN>, +@@ -31,7 +31,7 @@ + ) ENGINE=<STORAGE_ENGINE> <CUSTOM_TABLE_OPTIONS>; + SHOW KEYS IN t1; + Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +-t1 0 a 1 a # # NULL NULL # HASH ++t1 0 a 1 a # # NULL NULL # BTREE + INSERT INTO t1 (a,b) VALUES (1,'a'),(2,'b'); + INSERT INTO t1 (a,b) VALUES (1,'c'); + ERROR 23000: Duplicate entry '1' for key 'a' +@@ -43,7 +43,7 @@ + ALTER TABLE t1 ADD <CUSTOM_INDEX> (a) USING HASH COMMENT 'simple index on a'; + SHOW INDEX FROM t1; + Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +-t1 1 a 1 a # # NULL NULL # HASH simple index on a ++t1 1 a 1 a # # NULL NULL # BTREE simple index on a + ALTER TABLE t1 DROP KEY a; + DROP TABLE t1; + CREATE TABLE t1 (a <INT_COLUMN>, +@@ -52,7 +52,7 @@ + ) ENGINE=<STORAGE_ENGINE> <CUSTOM_TABLE_OPTIONS>; + SHOW KEYS IN t1; + Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +-t1 0 a 1 a # # NULL NULL # HASH ++t1 0 a 1 a # # NULL NULL # BTREE + INSERT INTO t1 (a,b) VALUES (1,'a'),(2,'b'); + INSERT INTO t1 (a,b) VALUES (1,'c'); + ERROR 23000: Duplicate entry '1' for key 'a' diff --git a/storage/myisam/mysql-test/storage_engine/misc.rdiff b/storage/myisam/mysql-test/storage_engine/misc.rdiff new file mode 100644 index 00000000..cdbad003 --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/misc.rdiff @@ -0,0 +1,34 @@ +--- suite/storage_engine/misc.result 2018-02-23 03:01:49.673249912 +0200 ++++ suite/storage_engine/misc.reject 2018-02-23 03:02:05.669249564 +0200 +@@ -28,6 +28,10 @@ + SELECT TABLE_NAME, COLUMN_NAME, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME + FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE ORDER BY TABLE_NAME; + TABLE_NAME COLUMN_NAME REFERENCED_TABLE_NAME REFERENCED_COLUMN_NAME ++Warning 1286 Unknown storage engine 'InnoDB' ++Warning 1286 Unknown storage engine 'InnoDB' ++Warning 1286 Unknown storage engine 'InnoDB' ++Warnings: + column_stats column_name NULL NULL + column_stats db_name NULL NULL + column_stats table_name NULL NULL +@@ -58,12 +62,6 @@ + index_stats index_name NULL NULL + index_stats prefix_arity NULL NULL + index_stats table_name NULL NULL +-innodb_index_stats database_name NULL NULL +-innodb_index_stats index_name NULL NULL +-innodb_index_stats stat_name NULL NULL +-innodb_index_stats table_name NULL NULL +-innodb_table_stats database_name NULL NULL +-innodb_table_stats table_name NULL NULL + plugin name NULL NULL + proc db NULL NULL + proc name NULL NULL +@@ -94,7 +92,5 @@ + time_zone_transition Transition_time NULL NULL + time_zone_transition_type Time_zone_id NULL NULL + time_zone_transition_type Transition_type_id NULL NULL +-transaction_registry commit_id NULL NULL +-transaction_registry transaction_id NULL NULL + user Host NULL NULL + user User NULL NULL diff --git a/storage/myisam/mysql-test/storage_engine/parts/disabled.def b/storage/myisam/mysql-test/storage_engine/parts/disabled.def new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/parts/disabled.def diff --git a/storage/myisam/mysql-test/storage_engine/show_engine.rdiff b/storage/myisam/mysql-test/storage_engine/show_engine.rdiff new file mode 100644 index 00000000..4e662465 --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/show_engine.rdiff @@ -0,0 +1,10 @@ +--- suite/storage_engine/show_engine.result 2012-06-24 23:55:19.539380000 +0400 ++++ suite/storage_engine/show_engine.reject 2012-07-15 16:21:54.401939228 +0400 +@@ -4,7 +4,6 @@ + # volatile data (timestamps, memory info, etc.) + SHOW ENGINE <STORAGE_ENGINE> STATUS; + Type Name Status +-<STORAGE_ENGINE> ### Engine status, can be long and changeable ### + # For SHOW MUTEX even the number of lines is volatile, so the result logging is disabled, + # the test only checks that the command does not produce any errors + SHOW ENGINE <STORAGE_ENGINE> MUTEX; diff --git a/storage/myisam/mysql-test/storage_engine/tbl_opt_insert_method.rdiff b/storage/myisam/mysql-test/storage_engine/tbl_opt_insert_method.rdiff new file mode 100644 index 00000000..a8df8525 --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/tbl_opt_insert_method.rdiff @@ -0,0 +1,11 @@ +--- suite/storage_engine/tbl_opt_insert_method.result 2012-06-24 23:55:19.539380000 +0400 ++++ suite/storage_engine/tbl_opt_insert_method.reject 2012-07-15 16:21:56.381914337 +0400 +@@ -5,7 +5,7 @@ + t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` char(8) DEFAULT NULL +-) ENGINE=<STORAGE_ENGINE> DEFAULT CHARSET=latin1 INSERT_METHOD=FIRST ++) ENGINE=<STORAGE_ENGINE> DEFAULT CHARSET=latin1 + ALTER TABLE t1 INSERT_METHOD=NO; + SHOW CREATE TABLE t1; + Table Create Table diff --git a/storage/myisam/mysql-test/storage_engine/tbl_opt_union.rdiff b/storage/myisam/mysql-test/storage_engine/tbl_opt_union.rdiff new file mode 100644 index 00000000..4fa16452 --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/tbl_opt_union.rdiff @@ -0,0 +1,16 @@ +--- suite/storage_engine/tbl_opt_union.result 2012-06-24 23:55:19.539380000 +0400 ++++ suite/storage_engine/tbl_opt_union.reject 2012-07-15 16:21:58.121892463 +0400 +@@ -4,11 +4,11 @@ + Table Create Table + t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL +-) ENGINE=<STORAGE_ENGINE> DEFAULT CHARSET=latin1 UNION=(`child1`) ++) ENGINE=<STORAGE_ENGINE> DEFAULT CHARSET=latin1 + ALTER TABLE t1 UNION = (child1,child2); + SHOW CREATE TABLE t1; + Table Create Table + t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL +-) ENGINE=<STORAGE_ENGINE> DEFAULT CHARSET=latin1 UNION=(`child1`,`child2`) ++) ENGINE=<STORAGE_ENGINE> DEFAULT CHARSET=latin1 + DROP TABLE t1, child1, child2; diff --git a/storage/myisam/mysql-test/storage_engine/trx/cons_snapshot_repeatable_read.rdiff b/storage/myisam/mysql-test/storage_engine/trx/cons_snapshot_repeatable_read.rdiff new file mode 100644 index 00000000..b460b5b4 --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/trx/cons_snapshot_repeatable_read.rdiff @@ -0,0 +1,20 @@ +--- suite/storage_engine/trx/cons_snapshot_repeatable_read.result 2013-01-22 22:05:05.246633000 +0400 ++++ suite/storage_engine/trx/cons_snapshot_repeatable_read.reject 2013-01-23 02:44:05.336711176 +0400 +@@ -1,3 +1,9 @@ ++# -- WARNING ---------------------------------------------------------------- ++# According to I_S.ENGINES, MyISAM does not support transactions. ++# If it is true, the test will most likely fail; you can ++# either create an rdiff file, or add the test to disabled.def. ++# If transactions should be supported, check the data in Information Schema. ++# --------------------------------------------------------------------------- + DROP TABLE IF EXISTS t1; + connect con1,localhost,root,,; + connect con2,localhost,root,,; +@@ -11,6 +17,7 @@ + # If consistent read works on this isolation level (REPEATABLE READ), the following SELECT should not return the value we inserted (1) + SELECT a FROM t1; + a ++1 + COMMIT; + connection default; + disconnect con1; diff --git a/storage/myisam/mysql-test/storage_engine/trx/cons_snapshot_serializable.rdiff b/storage/myisam/mysql-test/storage_engine/trx/cons_snapshot_serializable.rdiff new file mode 100644 index 00000000..d5f1a030 --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/trx/cons_snapshot_serializable.rdiff @@ -0,0 +1,20 @@ +--- suite/storage_engine/trx/cons_snapshot_serializable.result 2013-01-22 22:05:05.246633000 +0400 ++++ suite/storage_engine/trx/cons_snapshot_serializable.reject 2013-01-23 02:44:05.928703734 +0400 +@@ -1,3 +1,9 @@ ++# -- WARNING ---------------------------------------------------------------- ++# According to I_S.ENGINES, MyISAM does not support transactions. ++# If it is true, the test will most likely fail; you can ++# either create an rdiff file, or add the test to disabled.def. ++# If transactions should be supported, check the data in Information Schema. ++# --------------------------------------------------------------------------- + DROP TABLE IF EXISTS t1; + connect con1,localhost,root,,; + connect con2,localhost,root,,; +@@ -11,6 +17,7 @@ + # If consistent read works on this isolation level (SERIALIZABLE), the following SELECT should not return the value we inserted (1) + SELECT a FROM t1; + a ++1 + COMMIT; + connection default; + disconnect con1; diff --git a/storage/myisam/mysql-test/storage_engine/trx/delete.rdiff b/storage/myisam/mysql-test/storage_engine/trx/delete.rdiff new file mode 100644 index 00000000..d7111ab2 --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/trx/delete.rdiff @@ -0,0 +1,50 @@ +--- suite/storage_engine/trx/delete.result 2012-07-12 23:06:18.946113626 +0400 ++++ suite/storage_engine/trx/delete.reject 2012-07-15 16:55:46.108397219 +0400 +@@ -1,3 +1,15 @@ ++# -- WARNING ---------------------------------------------------------------- ++# According to I_S.ENGINES, MyISAM does not support transactions. ++# If it is true, the test will most likely fail; you can ++# either create an rdiff file, or add the test to disabled.def. ++# If transactions should be supported, check the data in Information Schema. ++# --------------------------------------------------------------------------- ++# -- WARNING ---------------------------------------------------------------- ++# According to I_S.ENGINES, MyISAM does not support savepoints. ++# If it is true, the test will most likely fail; you can ++# either create an rdiff file (recommended), or add the test to disabled.def. ++# If savepoints should be supported, check the data in Information Schema. ++# --------------------------------------------------------------------------- + DROP TABLE IF EXISTS t1; + CREATE TABLE t1 (a <INT_COLUMN>, b <CHAR_COLUMN>) ENGINE=<STORAGE_ENGINE> <CUSTOM_TABLE_OPTIONS>; + INSERT INTO t1 (a,b) VALUES (1,'a'),(2,'b'),(3,'c'),(4,'d'),(5,'e'),(6,'f'),(7,'g'),(8,'h'),(10000,'foobar'); +@@ -46,27 +58,17 @@ + DELETE FROM t1; + RELEASE SAVEPOINT spt1; + ROLLBACK; ++Warnings: ++Warning 1196 Some non-transactional changed tables couldn't be rolled back + SELECT a,b FROM t1; + a b +-10000 foobar +-10000 foobar +-2 b +-2 b +-4 d +-4 d +-5 e +-5 e +-6 f +-6 f +-7 g +-7 g +-8 h +-8 h + BEGIN; + DELETE FROM t1 WHERE a <= 4 ORDER BY b DESC LIMIT 1; + SAVEPOINT spt1; + DELETE FROM t1; + INSERT INTO t1 (a,b) VALUES (1,'a'); + ROLLBACK TO SAVEPOINT spt1; ++Warnings: ++Warning 1196 Some non-transactional changed tables couldn't be rolled back + COMMIT; + DROP TABLE t1; diff --git a/storage/myisam/mysql-test/storage_engine/trx/insert.rdiff b/storage/myisam/mysql-test/storage_engine/trx/insert.rdiff new file mode 100644 index 00000000..cee69633 --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/trx/insert.rdiff @@ -0,0 +1,65 @@ +--- suite/storage_engine/trx/insert.result 2012-07-12 23:09:44.663527407 +0400 ++++ suite/storage_engine/trx/insert.reject 2012-07-15 16:55:46.676390078 +0400 +@@ -1,3 +1,15 @@ ++# -- WARNING ---------------------------------------------------------------- ++# According to I_S.ENGINES, MyISAM does not support transactions. ++# If it is true, the test will most likely fail; you can ++# either create an rdiff file, or add the test to disabled.def. ++# If transactions should be supported, check the data in Information Schema. ++# --------------------------------------------------------------------------- ++# -- WARNING ---------------------------------------------------------------- ++# According to I_S.ENGINES, MyISAM does not support savepoints. ++# If it is true, the test will most likely fail; you can ++# either create an rdiff file (recommended), or add the test to disabled.def. ++# If savepoints should be supported, check the data in Information Schema. ++# --------------------------------------------------------------------------- + DROP TABLE IF EXISTS t1; + CREATE TABLE t1 (a <INT_COLUMN>, b <CHAR_COLUMN>) ENGINE=<STORAGE_ENGINE> <CUSTOM_TABLE_OPTIONS>; + BEGIN; +@@ -21,8 +33,11 @@ + RELEASE SAVEPOINT spt1; + INSERT INTO t1 (a,b) VALUES (DEFAULT,DEFAULT); + ROLLBACK; ++Warnings: ++Warning 1196 Some non-transactional changed tables couldn't be rolled back + SELECT a,b FROM t1; + a b ++0 test + 1 a + 10 foo + 100 foo +@@ -31,25 +46,34 @@ + 3 c + 4 d + 5 e ++NULL NULL ++NULL NULL + BEGIN; + INSERT t1 (a) VALUE (10),(20); + SAVEPOINT spt1; + INSERT INTO t1 SET a = 11, b = 'f'; + INSERT t1 SET b = DEFAULT; + ROLLBACK TO SAVEPOINT spt1; ++Warnings: ++Warning 1196 Some non-transactional changed tables couldn't be rolled back + INSERT INTO t1 (b,a) VALUES ('test1',10); + COMMIT; + SELECT a,b FROM t1; + a b ++0 test + 1 a + 10 NULL + 10 foo + 10 test1 + 100 foo + 11 abc ++11 f + 2 b + 20 NULL + 3 c + 4 d + 5 e ++NULL NULL ++NULL NULL ++NULL NULL + DROP TABLE t1; diff --git a/storage/myisam/mysql-test/storage_engine/trx/level_read_committed.rdiff b/storage/myisam/mysql-test/storage_engine/trx/level_read_committed.rdiff new file mode 100644 index 00000000..94e3fc83 --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/trx/level_read_committed.rdiff @@ -0,0 +1,94 @@ +--- suite/storage_engine/trx/level_read_committed.result 2013-01-22 22:05:05.246633000 +0400 ++++ suite/storage_engine/trx/level_read_committed.reject 2013-01-23 02:44:06.572695636 +0400 +@@ -1,3 +1,9 @@ ++# -- WARNING ---------------------------------------------------------------- ++# According to I_S.ENGINES, MyISAM does not support transactions. ++# If it is true, the test will most likely fail; you can ++# either create an rdiff file, or add the test to disabled.def. ++# If transactions should be supported, check the data in Information Schema. ++# --------------------------------------------------------------------------- + DROP TABLE IF EXISTS t1; + connect con1,localhost,root,,; + SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED; +@@ -16,6 +22,7 @@ + connection con1; + SELECT a FROM t1; + a ++1 + connection con2; + INSERT INTO t1 (a) VALUES (2); + # WARNING: Statement ended with errno 0, errname ''. +@@ -23,25 +30,37 @@ + connection con1; + SELECT a FROM t1; + a ++1 ++2 + INSERT INTO t1 (a) SELECT a+100 FROM t1; + # WARNING: Statement ended with errno 0, errname ''. + # If it differs from the result file, it might indicate a problem. + SELECT a FROM t1; + a ++1 ++101 ++102 ++2 + connection con2; + SELECT a FROM t1; + a + 1 ++101 ++102 + 2 + COMMIT; + SELECT a FROM t1; + a + 1 ++101 ++102 + 2 + connection con1; + SELECT a FROM t1; + a + 1 ++101 ++102 + 2 + INSERT INTO t1 (a) SELECT a+200 FROM t1; + # WARNING: Statement ended with errno 0, errname ''. +@@ -49,23 +68,35 @@ + SELECT a FROM t1; + a + 1 ++101 ++102 + 2 + 201 + 202 ++301 ++302 + COMMIT; + SELECT a FROM t1; + a + 1 ++101 ++102 + 2 + 201 + 202 ++301 ++302 + connection con2; + SELECT a FROM t1; + a + 1 ++101 ++102 + 2 + 201 + 202 ++301 ++302 + connection default; + disconnect con1; + disconnect con2; diff --git a/storage/myisam/mysql-test/storage_engine/trx/level_read_uncommitted.rdiff b/storage/myisam/mysql-test/storage_engine/trx/level_read_uncommitted.rdiff new file mode 100644 index 00000000..91a2786c --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/trx/level_read_uncommitted.rdiff @@ -0,0 +1,12 @@ +--- suite/storage_engine/trx/level_read_uncommitted.result 2013-01-22 22:05:05.246633000 +0400 ++++ suite/storage_engine/trx/level_read_uncommitted.reject 2013-01-23 02:44:07.196687792 +0400 +@@ -1,3 +1,9 @@ ++# -- WARNING ---------------------------------------------------------------- ++# According to I_S.ENGINES, MyISAM does not support transactions. ++# If it is true, the test will most likely fail; you can ++# either create an rdiff file, or add the test to disabled.def. ++# If transactions should be supported, check the data in Information Schema. ++# --------------------------------------------------------------------------- + DROP TABLE IF EXISTS t1; + connect con1,localhost,root,,; + SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; diff --git a/storage/myisam/mysql-test/storage_engine/trx/level_repeatable_read.rdiff b/storage/myisam/mysql-test/storage_engine/trx/level_repeatable_read.rdiff new file mode 100644 index 00000000..2c265900 --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/trx/level_repeatable_read.rdiff @@ -0,0 +1,96 @@ +--- suite/storage_engine/trx/level_repeatable_read.result 2013-01-22 22:05:05.246633000 +0400 ++++ suite/storage_engine/trx/level_repeatable_read.reject 2013-01-23 02:44:07.776680499 +0400 +@@ -1,3 +1,9 @@ ++# -- WARNING ---------------------------------------------------------------- ++# According to I_S.ENGINES, MyISAM does not support transactions. ++# If it is true, the test will most likely fail; you can ++# either create an rdiff file, or add the test to disabled.def. ++# If transactions should be supported, check the data in Information Schema. ++# --------------------------------------------------------------------------- + DROP TABLE IF EXISTS t1; + connect con1,localhost,root,,; + SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ; +@@ -16,6 +22,7 @@ + connection con1; + SELECT a FROM t1; + a ++1 + connection con2; + INSERT INTO t1 (a) VALUES (2); + # WARNING: Statement ended with errno 0, errname ''. +@@ -23,46 +30,73 @@ + connection con1; + SELECT a FROM t1; + a ++1 ++2 + INSERT INTO t1 (a) SELECT a+100 FROM t1; +-ERROR HY000: Lock wait timeout exceeded; try restarting transaction +-# WARNING: Statement ended with errno 1205, errname 'ER_LOCK_WAIT_TIMEOUT'. ++# WARNING: Statement ended with errno 0, errname ''. + # If it differs from the result file, it might indicate a problem. + SELECT a FROM t1; + a ++1 ++101 ++102 ++2 + connection con2; + SELECT a FROM t1; + a + 1 ++101 ++102 + 2 + COMMIT; + SELECT a FROM t1; + a + 1 ++101 ++102 + 2 + connection con1; + SELECT a FROM t1; + a ++1 ++101 ++102 ++2 + INSERT INTO t1 (a) SELECT a+200 FROM t1; + # WARNING: Statement ended with errno 0, errname ''. + # If it differs from the result file, it might indicate a problem. + SELECT a FROM t1; + a ++1 ++101 ++102 ++2 + 201 + 202 ++301 ++302 + COMMIT; + SELECT a FROM t1; + a + 1 ++101 ++102 + 2 + 201 + 202 ++301 ++302 + connection con2; + SELECT a FROM t1; + a + 1 ++101 ++102 + 2 + 201 + 202 ++301 ++302 + connection default; + disconnect con1; + disconnect con2; diff --git a/storage/myisam/mysql-test/storage_engine/trx/level_serializable.rdiff b/storage/myisam/mysql-test/storage_engine/trx/level_serializable.rdiff new file mode 100644 index 00000000..7955036e --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/trx/level_serializable.rdiff @@ -0,0 +1,103 @@ +--- suite/storage_engine/trx/level_serializable.result 2013-01-22 22:05:05.246633000 +0400 ++++ suite/storage_engine/trx/level_serializable.reject 2013-01-23 02:44:08.384672856 +0400 +@@ -1,3 +1,9 @@ ++# -- WARNING ---------------------------------------------------------------- ++# According to I_S.ENGINES, MyISAM does not support transactions. ++# If it is true, the test will most likely fail; you can ++# either create an rdiff file, or add the test to disabled.def. ++# If transactions should be supported, check the data in Information Schema. ++# --------------------------------------------------------------------------- + DROP TABLE IF EXISTS t1; + connect con1,localhost,root,,; + SET SESSION TRANSACTION ISOLATION LEVEL SERIALIZABLE; +@@ -11,45 +17,86 @@ + connection con2; + BEGIN; + INSERT INTO t1 (a) VALUES(1); +-ERROR HY000: Lock wait timeout exceeded; try restarting transaction +-# WARNING: Statement ended with errno 1205, errname 'ER_LOCK_WAIT_TIMEOUT'. ++# WARNING: Statement ended with errno 0, errname ''. + # If it differs from the result file, it might indicate a problem. + connection con1; + SELECT a FROM t1; + a ++1 + connection con2; + INSERT INTO t1 (a) VALUES (2); +-ERROR HY000: Lock wait timeout exceeded; try restarting transaction +-# WARNING: Statement ended with errno 1205, errname 'ER_LOCK_WAIT_TIMEOUT'. ++# WARNING: Statement ended with errno 0, errname ''. + # If it differs from the result file, it might indicate a problem. + connection con1; + SELECT a FROM t1; + a ++1 ++2 + INSERT INTO t1 (a) SELECT a+100 FROM t1; + # WARNING: Statement ended with errno 0, errname ''. + # If it differs from the result file, it might indicate a problem. + SELECT a FROM t1; + a ++1 ++101 ++102 ++2 + connection con2; + SELECT a FROM t1; + a ++1 ++101 ++102 ++2 + COMMIT; + SELECT a FROM t1; + a ++1 ++101 ++102 ++2 + connection con1; + SELECT a FROM t1; + a ++1 ++101 ++102 ++2 + INSERT INTO t1 (a) SELECT a+200 FROM t1; + # WARNING: Statement ended with errno 0, errname ''. + # If it differs from the result file, it might indicate a problem. + SELECT a FROM t1; + a ++1 ++101 ++102 ++2 ++201 ++202 ++301 ++302 + COMMIT; + SELECT a FROM t1; + a ++1 ++101 ++102 ++2 ++201 ++202 ++301 ++302 + connection con2; + SELECT a FROM t1; + a ++1 ++101 ++102 ++2 ++201 ++202 ++301 ++302 + connection default; + disconnect con1; + disconnect con2; diff --git a/storage/myisam/mysql-test/storage_engine/trx/select_for_update.rdiff b/storage/myisam/mysql-test/storage_engine/trx/select_for_update.rdiff new file mode 100644 index 00000000..044ce56d --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/trx/select_for_update.rdiff @@ -0,0 +1,50 @@ +--- suite/storage_engine/trx/select_for_update.result 2012-07-13 01:26:07.612653808 +0400 ++++ suite/storage_engine/trx/select_for_update.reject 2012-07-15 16:55:49.784351006 +0400 +@@ -1,3 +1,9 @@ ++# -- WARNING ---------------------------------------------------------------- ++# According to I_S.ENGINES, MyISAM does not support transactions. ++# If it is true, the test will most likely fail; you can ++# either create an rdiff file, or add the test to disabled.def. ++# If transactions should be supported, check the data in Information Schema. ++# --------------------------------------------------------------------------- + DROP TABLE IF EXISTS t1; + CREATE TABLE t1 (a <INT_COLUMN>, b <CHAR_COLUMN>) ENGINE=<STORAGE_ENGINE> <CUSTOM_TABLE_OPTIONS>; + INSERT INTO t1 (a,b) VALUES (1,'a'),(2,'b'),(3,'a'); +@@ -14,16 +20,33 @@ + 1 a + 3 a + SELECT a,b FROM t1 WHERE b='a' LOCK IN SHARE MODE; +-ERROR HY000: Lock wait timeout exceeded; try restarting transaction ++a b ++1 a ++3 a ++# ERROR: Statement succeeded (expected results: ER_LOCK_WAIT_TIMEOUT) ++# ------------ UNEXPECTED RESULT ------------ ++# The statement|command succeeded unexpectedly. ++# SELECT .. FOR UPDATE or LOCK IN SHARE MODE or the mix could be unsupported|malfunctioning, or the problem was caused by previous errors. ++# You can change the engine code, or create an rdiff, or disable the test by adding it to disabled.def. ++# Further in this test, the message might sometimes be suppressed; a part of the test might be skipped. ++# Also, this problem may cause a chain effect (more errors of different kinds in the test). ++# ------------------------------------------- + UPDATE t1 SET b='c' WHERE b='a'; +-ERROR HY000: Lock wait timeout exceeded; try restarting transaction ++# ERROR: Statement succeeded (expected results: ER_LOCK_WAIT_TIMEOUT) ++# ------------ UNEXPECTED RESULT ------------ ++# The statement|command succeeded unexpectedly. ++# UPDATE or SELECT .. FOR UPDATE or the mix could be unsupported|malfunctioning, or the problem was caused by previous errors. ++# You can change the engine code, or create an rdiff, or disable the test by adding it to disabled.def. ++# Further in this test, the message might sometimes be suppressed; a part of the test might be skipped. ++# Also, this problem may cause a chain effect (more errors of different kinds in the test). ++# ------------------------------------------- + connection con1; + COMMIT; + SELECT a,b FROM t1; + a b +-1 a ++1 c + 2 b +-3 a ++3 c + disconnect con1; + connection default; + UPDATE t1 SET b='c' WHERE b='a'; diff --git a/storage/myisam/mysql-test/storage_engine/trx/select_lock_in_share_mode.rdiff b/storage/myisam/mysql-test/storage_engine/trx/select_lock_in_share_mode.rdiff new file mode 100644 index 00000000..e1e8d305 --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/trx/select_lock_in_share_mode.rdiff @@ -0,0 +1,37 @@ +--- suite/storage_engine/trx/select_lock_in_share_mode.result 2012-07-13 01:30:17.505512229 +0400 ++++ suite/storage_engine/trx/select_lock_in_share_mode.reject 2012-07-15 16:55:50.444342708 +0400 +@@ -1,3 +1,9 @@ ++# -- WARNING ---------------------------------------------------------------- ++# According to I_S.ENGINES, MyISAM does not support transactions. ++# If it is true, the test will most likely fail; you can ++# either create an rdiff file, or add the test to disabled.def. ++# If transactions should be supported, check the data in Information Schema. ++# --------------------------------------------------------------------------- + DROP TABLE IF EXISTS t1; + CREATE TABLE t1 (a <INT_COLUMN>, b <CHAR_COLUMN>) ENGINE=<STORAGE_ENGINE> <CUSTOM_TABLE_OPTIONS>; + INSERT INTO t1 (a,b) VALUES (1,'a'),(2,'b'),(3,'a'); +@@ -18,14 +24,21 @@ + 1 a + 3 a + UPDATE t1 SET b='c' WHERE b='a'; +-ERROR HY000: Lock wait timeout exceeded; try restarting transaction ++# ERROR: Statement succeeded (expected results: ER_LOCK_WAIT_TIMEOUT) ++# ------------ UNEXPECTED RESULT ------------ ++# The statement|command succeeded unexpectedly. ++# LOCK IN SHARE MODE or UPDATE or the mix could be unsupported|malfunctioning, or the problem was caused by previous errors. ++# You can change the engine code, or create an rdiff, or disable the test by adding it to disabled.def. ++# Further in this test, the message might sometimes be suppressed; a part of the test might be skipped. ++# Also, this problem may cause a chain effect (more errors of different kinds in the test). ++# ------------------------------------------- + connection con1; + COMMIT; + SELECT a,b FROM t1; + a b +-1 a ++1 c + 2 b +-3 a ++3 c + disconnect con1; + connection default; + UPDATE t1 SET b='c' WHERE b='a'; diff --git a/storage/myisam/mysql-test/storage_engine/trx/update.rdiff b/storage/myisam/mysql-test/storage_engine/trx/update.rdiff new file mode 100644 index 00000000..ca3b77e7 --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/trx/update.rdiff @@ -0,0 +1,58 @@ +--- suite/storage_engine/trx/update.result 2012-07-13 01:43:50.355293322 +0400 ++++ suite/storage_engine/trx/update.reject 2012-07-15 16:55:51.016335518 +0400 +@@ -1,3 +1,15 @@ ++# -- WARNING ---------------------------------------------------------------- ++# According to I_S.ENGINES, MyISAM does not support transactions. ++# If it is true, the test will most likely fail; you can ++# either create an rdiff file, or add the test to disabled.def. ++# If transactions should be supported, check the data in Information Schema. ++# --------------------------------------------------------------------------- ++# -- WARNING ---------------------------------------------------------------- ++# According to I_S.ENGINES, MyISAM does not support savepoints. ++# If it is true, the test will most likely fail; you can ++# either create an rdiff file (recommended), or add the test to disabled.def. ++# If savepoints should be supported, check the data in Information Schema. ++# --------------------------------------------------------------------------- + DROP TABLE IF EXISTS t1; + CREATE TABLE t1 (a <INT_COLUMN>, b <CHAR_COLUMN>) ENGINE=<STORAGE_ENGINE> <CUSTOM_TABLE_OPTIONS>; + INSERT INTO t1 (a,b) VALUES (1,'a'),(2,'b'),(3,'c'),(4,'d'),(5,'e'),(10000,'foobar'); +@@ -24,25 +36,29 @@ + UPDATE t1 SET b = 'update' WHERE a <= 4 ORDER BY a DESC, b ASC LIMIT 3; + UPDATE t1 SET b = ''; + ROLLBACK; ++Warnings: ++Warning 1196 Some non-transactional changed tables couldn't be rolled back + BEGIN; + UPDATE t1 SET b = 'update2' WHERE a <= 100; + SAVEPOINT spt1; + UPDATE t1 SET b = ''; + ROLLBACK TO SAVEPOINT spt1; ++Warnings: ++Warning 1196 Some non-transactional changed tables couldn't be rolled back + UPDATE t1 SET b = 'upd' WHERE a = 10050; + COMMIT; + SELECT a,b FROM t1; + a b + 10050 upd + 10050 upd +-51 update2 +-51 update2 +-52 update2 +-52 update2 +-53 update2 +-53 update2 +-54 update2 +-54 update2 +-55 update2 +-55 update2 ++51 ++51 ++52 ++52 ++53 ++53 ++54 ++54 ++55 ++55 + DROP TABLE t1; diff --git a/storage/myisam/mysql-test/storage_engine/trx/xa.rdiff b/storage/myisam/mysql-test/storage_engine/trx/xa.rdiff new file mode 100644 index 00000000..73c3796b --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/trx/xa.rdiff @@ -0,0 +1,89 @@ +--- suite/storage_engine/trx/xa.result 2012-07-13 01:47:00.788899248 +0400 ++++ suite/storage_engine/trx/xa.reject 2012-07-15 16:55:51.604328125 +0400 +@@ -1,3 +1,9 @@ ++# -- WARNING ---------------------------------------------------------------- ++# According to I_S.ENGINES, MyISAM does not support XA. ++# If it is true, the test will most likely fail; you can ++# either create an rdiff file, or add the test to disabled.def. ++# If XA should be supported, check the data in Information Schema. ++# --------------------------------------------------------------------------- + DROP TABLE IF EXISTS t1; + connect con1,localhost,root,,; + connect con2,localhost,root,,; +@@ -9,17 +15,22 @@ + connection con1; + SELECT a FROM t1; + a ++1 + connection con2; + INSERT INTO t1 (a) VALUES (2); + XA END 'xa1'; + connection con1; + SELECT a FROM t1; + a ++1 ++2 + connection con2; + XA PREPARE 'xa1'; + connection con1; + SELECT a FROM t1; + a ++1 ++2 + connection con2; + XA RECOVER; + formatID gtrid_length bqual_length data +@@ -38,6 +49,7 @@ + a + 1 + 2 ++3 + connection con2; + INSERT INTO t1 (a) VALUES (4); + XA END 'xa2'; +@@ -46,6 +58,8 @@ + a + 1 + 2 ++3 ++4 + connection con2; + XA COMMIT 'xa2' ONE PHASE; + connection con1; +@@ -65,6 +79,7 @@ + 2 + 3 + 4 ++5 + connection con2; + INSERT INTO t1 (a) VALUES (6); + XA END 'xa3'; +@@ -75,6 +90,8 @@ + 2 + 3 + 4 ++5 ++6 + connection con2; + XA PREPARE 'xa3'; + connection con1; +@@ -84,8 +101,12 @@ + 2 + 3 + 4 ++5 ++6 + connection con2; + XA ROLLBACK 'xa3'; ++Warnings: ++Warning 1196 Some non-transactional changed tables couldn't be rolled back + connection con1; + SELECT a FROM t1; + a +@@ -93,4 +114,6 @@ + 2 + 3 + 4 ++5 ++6 + DROP TABLE t1; diff --git a/storage/myisam/mysql-test/storage_engine/trx/xa_recovery.rdiff b/storage/myisam/mysql-test/storage_engine/trx/xa_recovery.rdiff new file mode 100644 index 00000000..4b057019 --- /dev/null +++ b/storage/myisam/mysql-test/storage_engine/trx/xa_recovery.rdiff @@ -0,0 +1,32 @@ +--- suite/storage_engine/trx/xa_recovery.result 2012-07-13 01:48:46.859565758 +0400 ++++ suite/storage_engine/trx/xa_recovery.reject 2012-07-15 16:55:53.740301272 +0400 +@@ -1,3 +1,9 @@ ++# -- WARNING ---------------------------------------------------------------- ++# According to I_S.ENGINES, MyISAM does not support XA. ++# If it is true, the test will most likely fail; you can ++# either create an rdiff file, or add the test to disabled.def. ++# If XA should be supported, check the data in Information Schema. ++# --------------------------------------------------------------------------- + call mtr.add_suppression("Found 2 prepared XA transactions"); + FLUSH TABLES; + DROP TABLE IF EXISTS t1; +@@ -18,12 +24,17 @@ + connection default; + XA RECOVER; + formatID gtrid_length bqual_length data +-1 3 0 xa1 +-1 3 0 xa2 + XA ROLLBACK 'xa1'; ++ERROR XAE04: XAER_NOTA: Unknown XID + XA COMMIT 'xa2'; ++ERROR XAE04: XAER_NOTA: Unknown XID + SELECT a FROM t1; + a ++1 ++2 + 3 + 4 ++Warnings: ++Error 145 Table './test/t1' is marked as crashed and should be repaired ++Error 1034 1 client is using or hasn't closed the table properly + DROP TABLE t1; diff --git a/storage/myisam/rt_index.c b/storage/myisam/rt_index.c new file mode 100644 index 00000000..651e2e79 --- /dev/null +++ b/storage/myisam/rt_index.c @@ -0,0 +1,1126 @@ +/* Copyright (C) 2002-2006 MySQL AB & Ramil Kalimullin + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "myisamdef.h" + +#ifdef HAVE_RTREE_KEYS + +#include "rt_index.h" +#include "rt_key.h" +#include "rt_mbr.h" + +#define REINSERT_BUFFER_INC 10 +#define PICK_BY_AREA +/*#define PICK_BY_PERIMETER*/ + +typedef struct st_page_level +{ + uint level; + my_off_t offs; +} stPageLevel; + +typedef struct st_page_list +{ + ulong n_pages; + ulong m_pages; + stPageLevel *pages; +} stPageList; + + +/* + Find next key in r-tree according to search_flag recursively + + NOTES + Used in rtree_find_first() and rtree_find_next() + + RETURN + -1 Error + 0 Found + 1 Not found +*/ + +static int rtree_find_req(MI_INFO *info, MI_KEYDEF *keyinfo, uint search_flag, + uint nod_cmp_flag, my_off_t page, int level) +{ + uchar *k; + uchar *last; + uint nod_flag; + int res; + uchar *page_buf; + int k_len; + uint *saved_key = (uint*) (info->rtree_recursion_state) + level; + + if (!(page_buf = (uchar*)my_alloca((uint)keyinfo->block_length))) + { + my_errno = HA_ERR_OUT_OF_MEM; + return -1; + } + if (!_mi_fetch_keypage(info, keyinfo, page, DFLT_INIT_HITS, page_buf, 0)) + goto err1; + nod_flag = mi_test_if_nod(page_buf); + + k_len = keyinfo->keylength - info->s->base.rec_reflength; + + if(info->rtree_recursion_depth >= level) + { + k = page_buf + *saved_key; + } + else + { + k = rt_PAGE_FIRST_KEY(page_buf, nod_flag); + } + last = rt_PAGE_END(page_buf); + + for (; k < last; k = rt_PAGE_NEXT_KEY(k, k_len, nod_flag)) + { + if (nod_flag) + { + /* this is an internal node in the tree */ + if (!(res = rtree_key_cmp(keyinfo->seg, info->first_mbr_key, k, + info->last_rkey_length, nod_cmp_flag))) + { + switch ((res = rtree_find_req(info, keyinfo, search_flag, nod_cmp_flag, + _mi_kpos(nod_flag, k), level + 1))) + { + case 0: /* found - exit from recursion */ + *saved_key = (uint) (k - page_buf); + goto ok; + case 1: /* not found - continue searching */ + info->rtree_recursion_depth = level; + break; + default: /* error */ + case -1: + goto err1; + } + } + } + else + { + /* this is a leaf */ + if (!rtree_key_cmp(keyinfo->seg, info->first_mbr_key, k, + info->last_rkey_length, search_flag)) + { + uchar *after_key = rt_PAGE_NEXT_KEY(k, k_len, nod_flag); + info->lastpos = _mi_dpos(info, 0, after_key); + info->lastkey_length = k_len + info->s->base.rec_reflength; + memcpy(info->lastkey, k, info->lastkey_length); + info->rtree_recursion_depth = level; + *saved_key = (uint) (last - page_buf); + + if (after_key < last) + { + info->int_keypos = info->buff; + info->int_maxpos = info->buff + (last - after_key); + memcpy(info->buff, after_key, last - after_key); + info->buff_used = 0; + } + else + { + info->buff_used = 1; + } + + res = 0; + goto ok; + } + } + } + info->lastpos = HA_OFFSET_ERROR; + my_errno = HA_ERR_KEY_NOT_FOUND; + res = 1; + +ok: + my_afree((uchar*)page_buf); + return res; + +err1: + my_afree((uchar*)page_buf); + info->lastpos = HA_OFFSET_ERROR; + return -1; +} + + +/* + Find first key in r-tree according to search_flag condition + + SYNOPSIS + rtree_find_first() + info Handler to MyISAM file + uint keynr Key number to use + key Key to search for + key_length Length of 'key' + search_flag Bitmap of flags how to do the search + + RETURN + -1 Error + 0 Found + 1 Not found +*/ + +int rtree_find_first(MI_INFO *info, uint keynr, uchar *key, uint key_length, + uint search_flag) +{ + my_off_t root; + uint nod_cmp_flag; + MI_KEYDEF *keyinfo = info->s->keyinfo + keynr; + + /* + At the moment index can only properly handle the + MBR_INTERSECT, so we use it for all sorts of queries. + TODO: better searsh for CONTAINS/WITHIN. + */ + search_flag= nod_cmp_flag= MBR_INTERSECT; + + if ((root = info->s->state.key_root[keynr]) == HA_OFFSET_ERROR) + { + my_errno= HA_ERR_END_OF_FILE; + return -1; + } + + /* + Save searched key, include data pointer. + The data pointer is required if the search_flag contains MBR_DATA. + (minimum bounding rectangle) + */ + memcpy(info->first_mbr_key, key, keyinfo->keylength); + info->last_rkey_length = key_length; + + info->rtree_recursion_depth = -1; + info->buff_used = 1; + + /* + TODO better search for CONTAINS/WITHIN. + nod_cmp_flag= ((search_flag & (MBR_EQUAL | MBR_WITHIN)) ? + MBR_WITHIN : MBR_INTERSECT); + */ + return rtree_find_req(info, keyinfo, search_flag, nod_cmp_flag, root, 0); +} + + +/* + Find next key in r-tree according to search_flag condition + + SYNOPSIS + rtree_find_next() + info Handler to MyISAM file + uint keynr Key number to use + search_flag Bitmap of flags how to do the search + + RETURN + -1 Error + 0 Found + 1 Not found +*/ + +int rtree_find_next(MI_INFO *info, uint keynr, uint search_flag) +{ + my_off_t root; + uint nod_cmp_flag; + MI_KEYDEF *keyinfo = info->s->keyinfo + keynr; + /* + At the moment index can only properly handle the + MBR_INTERSECT, so we use it for all sorts of queries. + TODO: better searsh for CONTAINS/WITHIN. + */ + search_flag= nod_cmp_flag= MBR_INTERSECT; + + if (info->update & HA_STATE_DELETED) + return rtree_find_first(info, keynr, info->lastkey, info->lastkey_length, + search_flag); + + if (!info->buff_used) + { + uchar *key= info->int_keypos; + + while (key < info->int_maxpos) + { + if (!rtree_key_cmp(keyinfo->seg, info->first_mbr_key, key, + info->last_rkey_length, search_flag)) + { + uchar *after_key = key + keyinfo->keylength; + + info->lastpos= _mi_dpos(info, 0, after_key); + memcpy(info->lastkey, key, info->lastkey_length); + + if (after_key < info->int_maxpos) + info->int_keypos= after_key; + else + info->buff_used= 1; + return 0; + } + key+= keyinfo->keylength; + } + } + if ((root = info->s->state.key_root[keynr]) == HA_OFFSET_ERROR) + { + my_errno= HA_ERR_END_OF_FILE; + return -1; + } + + /* + TODO better search for CONTAINS/WITHIN. + nod_cmp_flag= (((search_flag & (MBR_EQUAL | MBR_WITHIN)) ? + MBR_WITHIN : MBR_INTERSECT)); + */ + return rtree_find_req(info, keyinfo, search_flag, nod_cmp_flag, root, 0); +} + + +/* + Get next key in r-tree recursively + + NOTES + Used in rtree_get_first() and rtree_get_next() + + RETURN + -1 Error + 0 Found + 1 Not found +*/ + +static int rtree_get_req(MI_INFO *info, MI_KEYDEF *keyinfo, uint key_length, + my_off_t page, int level) +{ + uchar *k; + uchar *last; + uint nod_flag; + int res; + uchar *page_buf; + uint k_len; + uint *saved_key = (uint*) (info->rtree_recursion_state) + level; + + if (!(page_buf = (uchar*)my_alloca((uint)keyinfo->block_length))) + return -1; + if (!_mi_fetch_keypage(info, keyinfo, page, DFLT_INIT_HITS, page_buf, 0)) + goto err1; + nod_flag = mi_test_if_nod(page_buf); + + k_len = keyinfo->keylength - info->s->base.rec_reflength; + + if(info->rtree_recursion_depth >= level) + { + k = page_buf + *saved_key; + if (!nod_flag) + { + /* Only leaf pages contain data references. */ + /* Need to check next key with data reference. */ + k = rt_PAGE_NEXT_KEY(k, k_len, nod_flag); + } + } + else + { + k = rt_PAGE_FIRST_KEY(page_buf, nod_flag); + } + last = rt_PAGE_END(page_buf); + + for (; k < last; k = rt_PAGE_NEXT_KEY(k, k_len, nod_flag)) + { + if (nod_flag) + { + /* this is an internal node in the tree */ + switch ((res = rtree_get_req(info, keyinfo, key_length, + _mi_kpos(nod_flag, k), level + 1))) + { + case 0: /* found - exit from recursion */ + *saved_key = (uint) (k - page_buf); + goto ok; + case 1: /* not found - continue searching */ + info->rtree_recursion_depth = level; + break; + default: + case -1: /* error */ + goto err1; + } + } + else + { + /* this is a leaf */ + uchar *after_key = rt_PAGE_NEXT_KEY(k, k_len, nod_flag); + info->lastpos = _mi_dpos(info, 0, after_key); + info->lastkey_length = k_len + info->s->base.rec_reflength; + memcpy(info->lastkey, k, info->lastkey_length); + + info->rtree_recursion_depth = level; + *saved_key = (uint) (k - page_buf); + + if (after_key < last) + { + info->int_keypos = (uchar*)saved_key; + memcpy(info->buff, page_buf, keyinfo->block_length); + info->int_maxpos = rt_PAGE_END(info->buff); + info->buff_used = 0; + } + else + { + info->buff_used = 1; + } + + res = 0; + goto ok; + } + } + info->lastpos = HA_OFFSET_ERROR; + my_errno = HA_ERR_KEY_NOT_FOUND; + res = 1; + +ok: + my_afree((uchar*)page_buf); + return res; + +err1: + my_afree((uchar*)page_buf); + info->lastpos = HA_OFFSET_ERROR; + return -1; +} + + +/* + Get first key in r-tree + + RETURN + -1 Error + 0 Found + 1 Not found +*/ + +int rtree_get_first(MI_INFO *info, uint keynr, uint key_length) +{ + my_off_t root; + MI_KEYDEF *keyinfo = info->s->keyinfo + keynr; + + if ((root = info->s->state.key_root[keynr]) == HA_OFFSET_ERROR) + { + my_errno= HA_ERR_END_OF_FILE; + return -1; + } + + info->rtree_recursion_depth = -1; + info->buff_used = 1; + + return rtree_get_req(info, keyinfo, key_length, root, 0); +} + + +/* + Get next key in r-tree + + RETURN + -1 Error + 0 Found + 1 Not found +*/ + +int rtree_get_next(MI_INFO *info, uint keynr, uint key_length) +{ + my_off_t root= info->s->state.key_root[keynr]; + MI_KEYDEF *keyinfo = info->s->keyinfo + keynr; + + if (root == HA_OFFSET_ERROR) + { + my_errno= HA_ERR_END_OF_FILE; + return -1; + } + + if (!info->buff_used && !info->page_changed) + { + uint k_len = keyinfo->keylength - info->s->base.rec_reflength; + /* rt_PAGE_NEXT_KEY(info->int_keypos) */ + uchar *key = info->buff + *(int*)info->int_keypos + k_len + + info->s->base.rec_reflength; + /* rt_PAGE_NEXT_KEY(key) */ + uchar *after_key = key + k_len + info->s->base.rec_reflength; + + info->lastpos = _mi_dpos(info, 0, after_key); + info->lastkey_length = k_len + info->s->base.rec_reflength; + memcpy(info->lastkey, key, k_len + info->s->base.rec_reflength); + + *(uint*)info->int_keypos = (uint) (key - info->buff); + if (after_key >= info->int_maxpos) + { + info->buff_used = 1; + } + + return 0; + } + + return rtree_get_req(info, keyinfo, key_length, root, 0); +} + + +/* + Choose non-leaf better key for insertion +*/ + +#ifdef PICK_BY_PERIMETER +static uchar *rtree_pick_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *key, + uint key_length, uchar *page_buf, uint nod_flag) +{ + double increase; + double best_incr = DBL_MAX; + double perimeter; + double UNINIT_VAR(best_perimeter); + uchar *UNINIT_VAR(best_key); + uchar *k = rt_PAGE_FIRST_KEY(page_buf, nod_flag); + uchar *last = rt_PAGE_END(page_buf); + + for (; k < last; k = rt_PAGE_NEXT_KEY(k, key_length, nod_flag)) + { + if ((increase = rtree_perimeter_increase(keyinfo->seg, k, key, key_length, + &perimeter)) == -1) + return NULL; + if ((increase < best_incr)|| + (increase == best_incr && perimeter < best_perimeter)) + { + best_key = k; + best_perimeter= perimeter; + best_incr = increase; + } + } + return best_key; +} + +#endif /*PICK_BY_PERIMETER*/ + +#ifdef PICK_BY_AREA +static uchar *rtree_pick_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *key, + uint key_length, uchar *page_buf, uint nod_flag) +{ + double increase; + double UNINIT_VAR(best_incr); + double area; + double UNINIT_VAR(best_area); + uchar *best_key= NULL; + uchar *k = rt_PAGE_FIRST_KEY(page_buf, nod_flag); + uchar *last = rt_PAGE_END(page_buf); + + for (; k < last; k = rt_PAGE_NEXT_KEY(k, key_length, nod_flag)) + { + /* The following is safe as -1.0 is an exact number */ + if ((increase = rtree_area_increase(keyinfo->seg, k, key, key_length, + &area)) == -1.0) + return NULL; + /* The following should be safe, even if we compare doubles */ + if (!best_key || increase < best_incr || + ((increase == best_incr) && (area < best_area))) + { + best_key = k; + best_area = area; + best_incr = increase; + } + } + return best_key; +} + +#endif /*PICK_BY_AREA*/ + +/* + Go down and insert key into tree + + RETURN + -1 Error + 0 Child was not split + 1 Child was split +*/ + +static int rtree_insert_req(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *key, + uint key_length, my_off_t page, my_off_t *new_page, + int ins_level, int level) +{ + uchar *k; + uint nod_flag; + uchar *page_buf; + int res; + DBUG_ENTER("rtree_insert_req"); + + if (!(page_buf = (uchar*)my_alloca((uint)keyinfo->block_length + + HA_MAX_KEY_BUFF))) + { + my_errno = HA_ERR_OUT_OF_MEM; + DBUG_RETURN(-1); /* purecov: inspected */ + } + if (!_mi_fetch_keypage(info, keyinfo, page, DFLT_INIT_HITS, page_buf, 0)) + goto err1; + nod_flag = mi_test_if_nod(page_buf); + DBUG_PRINT("rtree", ("page: %lu level: %d ins_level: %d nod_flag: %u", + (ulong) page, level, ins_level, nod_flag)); + + if ((ins_level == -1 && nod_flag) || /* key: go down to leaf */ + (ins_level > -1 && ins_level > level)) /* branch: go down to ins_level */ + { + if ((k = rtree_pick_key(info, keyinfo, key, key_length, page_buf, + nod_flag)) == NULL) + goto err1; + switch ((res = rtree_insert_req(info, keyinfo, key, key_length, + _mi_kpos(nod_flag, k), new_page, ins_level, level + 1))) + { + case 0: /* child was not split */ + { + rtree_combine_rect(keyinfo->seg, k, key, k, key_length); + if (_mi_write_keypage(info, keyinfo, page, DFLT_INIT_HITS, page_buf)) + goto err1; + goto ok; + } + case 1: /* child was split */ + { + uchar *new_key = page_buf + keyinfo->block_length + nod_flag; + /* set proper MBR for key */ + if (rtree_set_key_mbr(info, keyinfo, k, key_length, + _mi_kpos(nod_flag, k))) + goto err1; + /* add new key for new page */ + _mi_kpointer(info, new_key - nod_flag, *new_page); + if (rtree_set_key_mbr(info, keyinfo, new_key, key_length, *new_page)) + goto err1; + res = rtree_add_key(info, keyinfo, new_key, key_length, + page_buf, new_page); + if (_mi_write_keypage(info, keyinfo, page, DFLT_INIT_HITS, page_buf)) + goto err1; + goto ok; + } + default: + case -1: /* error */ + { + goto err1; + } + } + } + else + { + res = rtree_add_key(info, keyinfo, key, key_length, page_buf, new_page); + if (_mi_write_keypage(info, keyinfo, page, DFLT_INIT_HITS, page_buf)) + goto err1; + goto ok; + } + +ok: + my_afree((uchar*)page_buf); + DBUG_RETURN(res); + +err1: + my_afree((uchar*)page_buf); + DBUG_RETURN(-1); /* purecov: inspected */ +} + + +/* + Insert key into the tree + + RETURN + -1 Error + 0 Root was not split + 1 Root was split +*/ + +static int rtree_insert_level(MI_INFO *info, uint keynr, uchar *key, + uint key_length, int ins_level) +{ + my_off_t old_root; + MI_KEYDEF *keyinfo = info->s->keyinfo + keynr; + int res; + my_off_t new_page; + DBUG_ENTER("rtree_insert_level"); + + if ((old_root = info->s->state.key_root[keynr]) == HA_OFFSET_ERROR) + { + if ((old_root = _mi_new(info, keyinfo, DFLT_INIT_HITS)) == HA_OFFSET_ERROR) + DBUG_RETURN(-1); + info->buff_used = 1; + mi_putint(info->buff, 2, 0); + res = rtree_add_key(info, keyinfo, key, key_length, info->buff, NULL); + if (_mi_write_keypage(info, keyinfo, old_root, DFLT_INIT_HITS, info->buff)) + DBUG_RETURN(1); + info->s->state.key_root[keynr] = old_root; + DBUG_RETURN(res); + } + + switch ((res = rtree_insert_req(info, keyinfo, key, key_length, + old_root, &new_page, ins_level, 0))) + { + case 0: /* root was not split */ + { + break; + } + case 1: /* root was split, grow a new root */ + { + uchar *new_root_buf= info->buff + info->s->base.max_key_block_length; + my_off_t new_root; + uchar *new_key; + uint nod_flag = info->s->base.key_reflength; + + DBUG_PRINT("rtree", ("root was split, grow a new root")); + + mi_putint(new_root_buf, 2, nod_flag); + if ((new_root = _mi_new(info, keyinfo, DFLT_INIT_HITS)) == + HA_OFFSET_ERROR) + goto err1; + + new_key = new_root_buf + keyinfo->block_length + nod_flag; + + _mi_kpointer(info, new_key - nod_flag, old_root); + if (rtree_set_key_mbr(info, keyinfo, new_key, key_length, old_root)) + goto err1; + if (rtree_add_key(info, keyinfo, new_key, key_length, new_root_buf, NULL) + == -1) + goto err1; + _mi_kpointer(info, new_key - nod_flag, new_page); + if (rtree_set_key_mbr(info, keyinfo, new_key, key_length, new_page)) + goto err1; + if (rtree_add_key(info, keyinfo, new_key, key_length, new_root_buf, NULL) + == -1) + goto err1; + if (_mi_write_keypage(info, keyinfo, new_root, + DFLT_INIT_HITS, new_root_buf)) + goto err1; + info->s->state.key_root[keynr] = new_root; + DBUG_PRINT("rtree", ("new root page: %lu level: %d nod_flag: %u", + (ulong) new_root, 0, mi_test_if_nod(new_root_buf))); + + break; +err1: + DBUG_RETURN(-1); /* purecov: inspected */ + } + default: + case -1: /* error */ + { + break; + } + } + DBUG_RETURN(res); +} + + +/* + Insert key into the tree - interface function + + RETURN + -1 Error + 0 OK +*/ + +int rtree_insert(MI_INFO *info, uint keynr, uchar *key, uint key_length) +{ + DBUG_ENTER("rtree_insert"); + DBUG_RETURN((!key_length || + (rtree_insert_level(info, keynr, key, key_length, -1) == -1)) ? + -1 : 0); +} + + +/* + Fill reinsert page buffer + + RETURN + -1 Error + 0 OK +*/ + +static int rtree_fill_reinsert_list(stPageList *ReinsertList, my_off_t page, + int level) +{ + DBUG_ENTER("rtree_fill_reinsert_list"); + DBUG_PRINT("rtree", ("page: %lu level: %d", (ulong) page, level)); + if (ReinsertList->n_pages == ReinsertList->m_pages) + { + ReinsertList->m_pages += REINSERT_BUFFER_INC; + if (!(ReinsertList->pages = (stPageLevel*) + my_realloc(mi_key_memory_stPageList_pages, + (uchar*)ReinsertList->pages, + ReinsertList->m_pages * sizeof(stPageLevel), + MYF(MY_ALLOW_ZERO_PTR)))) + goto err1; + } + /* save page to ReinsertList */ + ReinsertList->pages[ReinsertList->n_pages].offs = page; + ReinsertList->pages[ReinsertList->n_pages].level = level; + ReinsertList->n_pages++; + DBUG_RETURN(0); + +err1: + DBUG_RETURN(-1); /* purecov: inspected */ +} + + +/* + Go down and delete key from the tree + + RETURN + -1 Error + 0 Deleted + 1 Not found + 2 Empty leaf +*/ + +static int rtree_delete_req(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *key, + uint key_length, my_off_t page, uint *page_size, + stPageList *ReinsertList, int level) +{ + uchar *k; + uchar *last; + ulong i; + uint nod_flag; + uchar *page_buf; + int res; + DBUG_ENTER("rtree_delete_req"); + + if (!(page_buf = (uchar*)my_alloca((uint)keyinfo->block_length))) + { + my_errno = HA_ERR_OUT_OF_MEM; + DBUG_RETURN(-1); /* purecov: inspected */ + } + if (!_mi_fetch_keypage(info, keyinfo, page, DFLT_INIT_HITS, page_buf, 0)) + goto err1; + nod_flag = mi_test_if_nod(page_buf); + DBUG_PRINT("rtree", ("page: %lu level: %d nod_flag: %u", + (ulong) page, level, nod_flag)); + + k = rt_PAGE_FIRST_KEY(page_buf, nod_flag); + last = rt_PAGE_END(page_buf); + + for (i = 0; k < last; k = rt_PAGE_NEXT_KEY(k, key_length, nod_flag), ++i) + { + if (nod_flag) + { + /* not leaf */ + if (!rtree_key_cmp(keyinfo->seg, key, k, key_length, MBR_WITHIN)) + { + switch ((res = rtree_delete_req(info, keyinfo, key, key_length, + _mi_kpos(nod_flag, k), page_size, ReinsertList, level + 1))) + { + case 0: /* deleted */ + { + /* test page filling */ + if (*page_size + key_length >= rt_PAGE_MIN_SIZE(keyinfo->block_length)) + { + /* OK */ + /* Calculate a new key value (MBR) for the shrinked block. */ + if (rtree_set_key_mbr(info, keyinfo, k, key_length, + _mi_kpos(nod_flag, k))) + goto err1; + if (_mi_write_keypage(info, keyinfo, page, + DFLT_INIT_HITS, page_buf)) + goto err1; + } + else + { + /* + Too small: delete key & add it descendant to reinsert list. + Store position and level of the block so that it can be + accessed later for inserting the remaining keys. + */ + DBUG_PRINT("rtree", ("too small. move block to reinsert list")); + if (rtree_fill_reinsert_list(ReinsertList, _mi_kpos(nod_flag, k), + level + 1)) + goto err1; + /* + Delete the key that references the block. This makes the + block disappear from the index. Hence we need to insert + its remaining keys later. Note: if the block is a branch + block, we do not only remove this block, but the whole + subtree. So we need to re-insert its keys on the same + level later to reintegrate the subtrees. + */ + rtree_delete_key(info, page_buf, k, key_length, nod_flag); + if (_mi_write_keypage(info, keyinfo, page, + DFLT_INIT_HITS, page_buf)) + goto err1; + *page_size = mi_getint(page_buf); + } + + goto ok; + } + case 1: /* not found - continue searching */ + { + break; + } + case 2: /* vacuous case: last key in the leaf */ + { + rtree_delete_key(info, page_buf, k, key_length, nod_flag); + if (_mi_write_keypage(info, keyinfo, page, + DFLT_INIT_HITS, page_buf)) + goto err1; + *page_size = mi_getint(page_buf); + res = 0; + goto ok; + } + default: /* error */ + case -1: + { + goto err1; + } + } + } + } + else + { + /* leaf */ + if (!rtree_key_cmp(keyinfo->seg, key, k, key_length, MBR_EQUAL | MBR_DATA)) + { + rtree_delete_key(info, page_buf, k, key_length, nod_flag); + *page_size = mi_getint(page_buf); + if (*page_size == 2) + { + /* last key in the leaf */ + res = 2; + if (_mi_dispose(info, keyinfo, page, DFLT_INIT_HITS)) + goto err1; + } + else + { + res = 0; + if (_mi_write_keypage(info, keyinfo, page, DFLT_INIT_HITS, page_buf)) + goto err1; + } + goto ok; + } + } + } + res = 1; + +ok: + my_afree((uchar*)page_buf); + DBUG_RETURN(res); + +err1: + my_afree((uchar*)page_buf); + DBUG_RETURN(-1); /* purecov: inspected */ +} + + +/* + Delete key - interface function + + RETURN + -1 Error + 0 Deleted +*/ + +int rtree_delete(MI_INFO *info, uint keynr, uchar *key, uint key_length) +{ + uint page_size; + stPageList ReinsertList; + my_off_t old_root; + MI_KEYDEF *keyinfo = info->s->keyinfo + keynr; + DBUG_ENTER("rtree_delete"); + + if ((old_root = info->s->state.key_root[keynr]) == HA_OFFSET_ERROR) + { + my_errno= HA_ERR_END_OF_FILE; + DBUG_RETURN(-1); /* purecov: inspected */ + } + DBUG_PRINT("rtree", ("starting deletion at root page: %lu", + (ulong) old_root)); + + ReinsertList.pages = NULL; + ReinsertList.n_pages = 0; + ReinsertList.m_pages = 0; + + switch (rtree_delete_req(info, keyinfo, key, key_length, old_root, + &page_size, &ReinsertList, 0)) + { + case 2: /* empty */ + { + info->s->state.key_root[keynr] = HA_OFFSET_ERROR; + DBUG_RETURN(0); + } + case 0: /* deleted */ + { + uint nod_flag; + ulong i; + for (i = 0; i < ReinsertList.n_pages; ++i) + { + uchar *page_buf; + uchar *k; + uchar *last; + + if (!(page_buf = (uchar*)my_alloca((uint)keyinfo->block_length))) + { + my_errno = HA_ERR_OUT_OF_MEM; + goto err1; + } + if (!_mi_fetch_keypage(info, keyinfo, ReinsertList.pages[i].offs, + DFLT_INIT_HITS, page_buf, 0)) + goto err1; + nod_flag = mi_test_if_nod(page_buf); + DBUG_PRINT("rtree", ("reinserting keys from " + "page: %lu level: %d nod_flag: %u", + (ulong) ReinsertList.pages[i].offs, + ReinsertList.pages[i].level, nod_flag)); + + k = rt_PAGE_FIRST_KEY(page_buf, nod_flag); + last = rt_PAGE_END(page_buf); + for (; k < last; k = rt_PAGE_NEXT_KEY(k, key_length, nod_flag)) + { + int res; + if ((res= rtree_insert_level(info, keynr, k, key_length, + ReinsertList.pages[i].level)) == -1) + { + my_afree((uchar*)page_buf); + goto err1; + } + if (res) + { + ulong j; + DBUG_PRINT("rtree", ("root has been split, adjust levels")); + for (j= i; j < ReinsertList.n_pages; j++) + { + ReinsertList.pages[j].level++; + DBUG_PRINT("rtree", ("keys from page: %lu now level: %d", + (ulong) ReinsertList.pages[i].offs, + ReinsertList.pages[i].level)); + } + } + } + my_afree((uchar*)page_buf); + if (_mi_dispose(info, keyinfo, ReinsertList.pages[i].offs, + DFLT_INIT_HITS)) + goto err1; + } + if (ReinsertList.pages) + my_free(ReinsertList.pages); + + /* check for redundant root (not leaf, 1 child) and eliminate */ + if ((old_root = info->s->state.key_root[keynr]) == HA_OFFSET_ERROR) + goto err1; + if (!_mi_fetch_keypage(info, keyinfo, old_root, DFLT_INIT_HITS, + info->buff, 0)) + goto err1; + nod_flag = mi_test_if_nod(info->buff); + page_size = mi_getint(info->buff); + if (nod_flag && (page_size == 2 + key_length + nod_flag)) + { + my_off_t new_root = _mi_kpos(nod_flag, + rt_PAGE_FIRST_KEY(info->buff, nod_flag)); + if (_mi_dispose(info, keyinfo, old_root, DFLT_INIT_HITS)) + goto err1; + info->s->state.key_root[keynr] = new_root; + } + info->update= HA_STATE_DELETED; + DBUG_RETURN(0); + +err1: + DBUG_RETURN(-1); /* purecov: inspected */ + } + case 1: /* not found */ + { + my_errno = HA_ERR_KEY_NOT_FOUND; + DBUG_RETURN(-1); /* purecov: inspected */ + } + default: + case -1: /* error */ + { + DBUG_RETURN(-1); /* purecov: inspected */ + } + } +} + + +/* + Estimate number of suitable keys in the tree + + RETURN + estimated value +*/ + +ha_rows rtree_estimate(MI_INFO *info, uint keynr, uchar *key, + uint key_length, uint flag) +{ + MI_KEYDEF *keyinfo = info->s->keyinfo + keynr; + my_off_t root; + uint i = 0; + uchar *k; + uchar *last; + uint nod_flag; + uchar *page_buf; + uint k_len; + double area = 0; + ha_rows res = 0; + + if (flag & MBR_DISJOINT) + return HA_POS_ERROR; + + if ((root = info->s->state.key_root[keynr]) == HA_OFFSET_ERROR) + return HA_POS_ERROR; + if (!(page_buf = (uchar*)my_alloca((uint)keyinfo->block_length))) + return HA_POS_ERROR; + if (!_mi_fetch_keypage(info, keyinfo, root, DFLT_INIT_HITS, page_buf, 0)) + goto err1; + nod_flag = mi_test_if_nod(page_buf); + + k_len = keyinfo->keylength - info->s->base.rec_reflength; + + k = rt_PAGE_FIRST_KEY(page_buf, nod_flag); + last = rt_PAGE_END(page_buf); + + for (; k < last; k = rt_PAGE_NEXT_KEY(k, k_len, nod_flag), ++i) + { + if (nod_flag) + { + double k_area = rtree_rect_volume(keyinfo->seg, k, key_length); + + /* The following should be safe, even if we compare doubles */ + if (k_area == 0) + { + if (flag & (MBR_CONTAIN | MBR_INTERSECT)) + { + area += 1; + } + else if (flag & (MBR_WITHIN | MBR_EQUAL)) + { + if (!rtree_key_cmp(keyinfo->seg, key, k, key_length, MBR_WITHIN)) + area += 1; + } + else + goto err1; + } + else + { + if (flag & (MBR_CONTAIN | MBR_INTERSECT)) + { + area += rtree_overlapping_area(keyinfo->seg, key, k, key_length) / + k_area; + } + else if (flag & (MBR_WITHIN | MBR_EQUAL)) + { + if (!rtree_key_cmp(keyinfo->seg, key, k, key_length, MBR_WITHIN)) + area += rtree_rect_volume(keyinfo->seg, key, key_length) / + k_area; + } + else + goto err1; + } + } + else + { + if (!rtree_key_cmp(keyinfo->seg, key, k, key_length, flag)) + ++res; + } + } + if (nod_flag) + { + if (i) + res = (ha_rows) (area / i * info->state->records); + else + res = HA_POS_ERROR; + } + + my_afree((uchar*)page_buf); + return res; + +err1: + my_afree((uchar*)page_buf); + return HA_POS_ERROR; +} + +#endif /*HAVE_RTREE_KEYS*/ + diff --git a/storage/myisam/rt_index.h b/storage/myisam/rt_index.h new file mode 100644 index 00000000..2c100929 --- /dev/null +++ b/storage/myisam/rt_index.h @@ -0,0 +1,46 @@ +/* Copyright (c) 2002, 2004-2006 MySQL AB + Use is subject to license terms + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +#ifndef _rt_index_h +#define _rt_index_h + +#ifdef HAVE_RTREE_KEYS + +#define rt_PAGE_FIRST_KEY(page, nod_flag) (page + 2 + nod_flag) +#define rt_PAGE_NEXT_KEY(key, key_length, nod_flag) (key + key_length + \ + (nod_flag ? nod_flag : info->s->base.rec_reflength)) +#define rt_PAGE_END(page) (page + mi_getint(page)) + +#define rt_PAGE_MIN_SIZE(block_length) ((uint)(block_length) / 3) + +int rtree_insert(MI_INFO *info, uint keynr, uchar *key, uint key_length); +int rtree_delete(MI_INFO *info, uint keynr, uchar *key, uint key_length); + +int rtree_find_first(MI_INFO *info, uint keynr, uchar *key, uint key_length, + uint search_flag); +int rtree_find_next(MI_INFO *info, uint keynr, uint search_flag); + +int rtree_get_first(MI_INFO *info, uint keynr, uint key_length); +int rtree_get_next(MI_INFO *info, uint keynr, uint key_length); + +ha_rows rtree_estimate(MI_INFO *info, uint keynr, uchar *key, + uint key_length, uint flag); + +int rtree_split_page(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, uchar *key, + uint key_length, my_off_t *new_page_offs); + +#endif /*HAVE_RTREE_KEYS*/ +#endif /* _rt_index_h */ diff --git a/storage/myisam/rt_key.c b/storage/myisam/rt_key.c new file mode 100644 index 00000000..3c58a454 --- /dev/null +++ b/storage/myisam/rt_key.c @@ -0,0 +1,107 @@ +/* Copyright (c) 2000, 2002-2005, 2007 MySQL AB + Use is subject to license terms + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "myisamdef.h" + +#ifdef HAVE_RTREE_KEYS +#include "rt_index.h" +#include "rt_key.h" +#include "rt_mbr.h" + +/* + Add key to the page + + RESULT VALUES + -1 Error + 0 Not split + 1 Split +*/ + +int rtree_add_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *key, + uint key_length, uchar *page_buf, my_off_t *new_page) +{ + uint page_size = mi_getint(page_buf); + uint nod_flag = mi_test_if_nod(page_buf); + DBUG_ENTER("rtree_add_key"); + + if (page_size + key_length + info->s->base.rec_reflength <= + keyinfo->block_length) + { + /* split won't be necessary */ + if (nod_flag) + { + /* save key */ + DBUG_ASSERT(_mi_kpos(nod_flag, key) < info->state->key_file_length); + memcpy(rt_PAGE_END(page_buf), key - nod_flag, key_length + nod_flag); + page_size += key_length + nod_flag; + } + else + { + /* save key */ + DBUG_ASSERT(_mi_dpos(info, nod_flag, key + key_length + + info->s->base.rec_reflength) < + info->state->data_file_length + info->s->base.pack_reclength); + memcpy(rt_PAGE_END(page_buf), key, key_length + + info->s->base.rec_reflength); + page_size += key_length + info->s->base.rec_reflength; + } + mi_putint(page_buf, page_size, nod_flag); + DBUG_RETURN(0); + } + + DBUG_RETURN((rtree_split_page(info, keyinfo, page_buf, key, key_length, + new_page) ? -1 : 1)); +} + +/* + Delete key from the page +*/ +int rtree_delete_key(MI_INFO *info, uchar *page_buf, uchar *key, + uint key_length, uint nod_flag) +{ + uint16 page_size = mi_getint(page_buf); + uchar *key_start; + + key_start= key - nod_flag; + if (!nod_flag) + key_length += info->s->base.rec_reflength; + + memmove(key_start, key + key_length, page_size - key_length - + (key - page_buf)); + page_size-= key_length + nod_flag; + + mi_putint(page_buf, page_size, nod_flag); + return 0; +} + + +/* + Calculate and store key MBR +*/ + +int rtree_set_key_mbr(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *key, + uint key_length, my_off_t child_page) +{ + DBUG_ENTER("rtree_set_key_mbr"); + + if (!_mi_fetch_keypage(info, keyinfo, child_page, + DFLT_INIT_HITS, info->buff, 0)) + DBUG_RETURN(-1); /* purecov: inspected */ + + DBUG_RETURN(rtree_page_mbr(info, keyinfo->seg, info->buff, key, key_length)); +} + +#endif /*HAVE_RTREE_KEYS*/ diff --git a/storage/myisam/rt_key.h b/storage/myisam/rt_key.h new file mode 100644 index 00000000..56f57d84 --- /dev/null +++ b/storage/myisam/rt_key.h @@ -0,0 +1,32 @@ +/* Copyright (c) 2002, 2004-2006 MySQL AB + Use is subject to license terms + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Written by Ramil Kalimullin, who has a shared copyright to this code */ + +#ifndef _rt_key_h +#define _rt_key_h + +#ifdef HAVE_RTREE_KEYS + +int rtree_add_key(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *key, + uint key_length, uchar *page_buf, my_off_t *new_page); +int rtree_delete_key(MI_INFO *info, uchar *page, uchar *key, + uint key_length, uint nod_flag); +int rtree_set_key_mbr(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *key, + uint key_length, my_off_t child_page); + +#endif /*HAVE_RTREE_KEYS*/ +#endif /* _rt_key_h */ diff --git a/storage/myisam/rt_mbr.c b/storage/myisam/rt_mbr.c new file mode 100644 index 00000000..8bcaf9f6 --- /dev/null +++ b/storage/myisam/rt_mbr.c @@ -0,0 +1,807 @@ +/* Copyright (c) 2002-2007 MySQL AB + Use is subject to license terms + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "myisamdef.h" + +#ifdef HAVE_RTREE_KEYS + +#include "rt_index.h" +#include "rt_mbr.h" + +#define INTERSECT_CMP(amin, amax, bmin, bmax) ((amin > bmax) || (bmin > amax)) +#define CONTAIN_CMP(amin, amax, bmin, bmax) ((bmin > amin) || (bmax < amax)) +#define WITHIN_CMP(amin, amax, bmin, bmax) ((amin > bmin) || (amax < bmax)) +#define DISJOINT_CMP(amin, amax, bmin, bmax) ((amin <= bmax) && (bmin <= amax)) +#define EQUAL_CMP(amin, amax, bmin, bmax) ((amin != bmin) || (amax != bmax)) + +#define FCMP(A, B) ((int)(A) - (int)(B)) +#define p_inc(A, B, X) {A += X; B += X;} + +#define RT_CMP(nextflag) \ + if (nextflag & MBR_INTERSECT) \ + { \ + if (INTERSECT_CMP(amin, amax, bmin, bmax)) \ + return 1; \ + } \ + else if (nextflag & MBR_CONTAIN) \ + { \ + if (CONTAIN_CMP(amin, amax, bmin, bmax)) \ + return 1; \ + } \ + else if (nextflag & MBR_WITHIN) \ + { \ + if (WITHIN_CMP(amin, amax, bmin, bmax)) \ + return 1; \ + } \ + else if (nextflag & MBR_EQUAL) \ + { \ + if (EQUAL_CMP(amin, amax, bmin, bmax)) \ + return 1; \ + } \ + else if (nextflag & MBR_DISJOINT) \ + { \ + if (DISJOINT_CMP(amin, amax, bmin, bmax)) \ + return 1; \ + }\ + else /* if unknown comparison operator */ \ + { \ + DBUG_ASSERT(0); \ + } + +#define RT_CMP_KORR(type, korr_func, len, nextflag) \ +{ \ + type amin, amax, bmin, bmax; \ + amin = korr_func(a); \ + bmin = korr_func(b); \ + amax = korr_func(a+len); \ + bmax = korr_func(b+len); \ + RT_CMP(nextflag); \ +} + +#define RT_CMP_GET(type, get_func, len, nextflag) \ +{ \ + type amin, amax, bmin, bmax; \ + get_func(amin, a); \ + get_func(bmin, b); \ + get_func(amax, a+len); \ + get_func(bmax, b+len); \ + RT_CMP(nextflag); \ +} + +/* + Compares two keys a and b depending on nextflag + nextflag can contain these flags: + MBR_INTERSECT(a,b) a overlaps b + MBR_CONTAIN(a,b) a contains b + MBR_DISJOINT(a,b) a disjoint b + MBR_WITHIN(a,b) a within b + MBR_EQUAL(a,b) All coordinates of MBRs are equal + MBR_DATA(a,b) Data reference is the same + Returns 0 on success. +*/ +int rtree_key_cmp(HA_KEYSEG *keyseg, uchar *b, uchar *a, uint key_length, + uint nextflag) +{ + for (; (int) key_length > 0; keyseg += 2 ) + { + uint32 keyseg_length; + switch ((enum ha_base_keytype) keyseg->type) { + case HA_KEYTYPE_INT8: + RT_CMP_KORR(int8, mi_sint1korr, 1, nextflag); + break; + case HA_KEYTYPE_BINARY: + RT_CMP_KORR(uint8, mi_uint1korr, 1, nextflag); + break; + case HA_KEYTYPE_SHORT_INT: + RT_CMP_KORR(int16, mi_sint2korr, 2, nextflag); + break; + case HA_KEYTYPE_USHORT_INT: + RT_CMP_KORR(uint16, mi_uint2korr, 2, nextflag); + break; + case HA_KEYTYPE_INT24: + RT_CMP_KORR(int32, mi_sint3korr, 3, nextflag); + break; + case HA_KEYTYPE_UINT24: + RT_CMP_KORR(uint32, mi_uint3korr, 3, nextflag); + break; + case HA_KEYTYPE_LONG_INT: + RT_CMP_KORR(int32, mi_sint4korr, 4, nextflag); + break; + case HA_KEYTYPE_ULONG_INT: + RT_CMP_KORR(uint32, mi_uint4korr, 4, nextflag); + break; +#ifdef HAVE_LONG_LONG + case HA_KEYTYPE_LONGLONG: + RT_CMP_KORR(longlong, mi_sint8korr, 8, nextflag) + break; + case HA_KEYTYPE_ULONGLONG: + RT_CMP_KORR(ulonglong, mi_uint8korr, 8, nextflag) + break; +#endif + case HA_KEYTYPE_FLOAT: + /* The following should be safe, even if we compare doubles */ + RT_CMP_GET(float, mi_float4get, 4, nextflag); + break; + case HA_KEYTYPE_DOUBLE: + RT_CMP_GET(double, mi_float8get, 8, nextflag); + break; + case HA_KEYTYPE_END: + goto end; + default: + return 1; + } + keyseg_length= keyseg->length * 2; + key_length-= keyseg_length; + a+= keyseg_length; + b+= keyseg_length; + } + +end: + if (nextflag & MBR_DATA) + { + uchar *end = a + keyseg->length; + do + { + if (*a++ != *b++) + return FCMP(a[-1], b[-1]); + } while (a != end); + } + return 0; +} + +#define RT_VOL_KORR(type, korr_func, len, cast) \ +{ \ + type amin, amax; \ + amin = korr_func(a); \ + amax = korr_func(a+len); \ + res *= (cast(amax) - cast(amin)); \ +} + +#define RT_VOL_GET(type, get_func, len, cast) \ +{ \ + type amin, amax; \ + get_func(amin, a); \ + get_func(amax, a+len); \ + res *= (cast(amax) - cast(amin)); \ +} + +/* + Calculates rectangle volume +*/ +double rtree_rect_volume(HA_KEYSEG *keyseg, uchar *a, uint key_length) +{ + double res = 1; + for (; (int)key_length > 0; keyseg += 2) + { + uint32 keyseg_length; + switch ((enum ha_base_keytype) keyseg->type) { + case HA_KEYTYPE_INT8: + RT_VOL_KORR(int8, mi_sint1korr, 1, (double)); + break; + case HA_KEYTYPE_BINARY: + RT_VOL_KORR(uint8, mi_uint1korr, 1, (double)); + break; + case HA_KEYTYPE_SHORT_INT: + RT_VOL_KORR(int16, mi_sint2korr, 2, (double)); + break; + case HA_KEYTYPE_USHORT_INT: + RT_VOL_KORR(uint16, mi_uint2korr, 2, (double)); + break; + case HA_KEYTYPE_INT24: + RT_VOL_KORR(int32, mi_sint3korr, 3, (double)); + break; + case HA_KEYTYPE_UINT24: + RT_VOL_KORR(uint32, mi_uint3korr, 3, (double)); + break; + case HA_KEYTYPE_LONG_INT: + RT_VOL_KORR(int32, mi_sint4korr, 4, (double)); + break; + case HA_KEYTYPE_ULONG_INT: + RT_VOL_KORR(uint32, mi_uint4korr, 4, (double)); + break; +#ifdef HAVE_LONG_LONG + case HA_KEYTYPE_LONGLONG: + RT_VOL_KORR(longlong, mi_sint8korr, 8, (double)); + break; + case HA_KEYTYPE_ULONGLONG: + RT_VOL_KORR(longlong, mi_sint8korr, 8, ulonglong2double); + break; +#endif + case HA_KEYTYPE_FLOAT: + RT_VOL_GET(float, mi_float4get, 4, (double)); + break; + case HA_KEYTYPE_DOUBLE: + RT_VOL_GET(double, mi_float8get, 8, (double)); + break; + case HA_KEYTYPE_END: + key_length = 0; + break; + default: + return -1; + } + keyseg_length= keyseg->length * 2; + key_length-= keyseg_length; + a+= keyseg_length; + } + return res; +} + +#define RT_D_MBR_KORR(type, korr_func, len, cast) \ +{ \ + type amin, amax; \ + amin = korr_func(a); \ + amax = korr_func(a+len); \ + *res++ = cast(amin); \ + *res++ = cast(amax); \ +} + +#define RT_D_MBR_GET(type, get_func, len, cast) \ +{ \ + type amin, amax; \ + get_func(amin, a); \ + get_func(amax, a+len); \ + *res++ = cast(amin); \ + *res++ = cast(amax); \ +} + + +/* + Creates an MBR as an array of doubles. +*/ + +int rtree_d_mbr(HA_KEYSEG *keyseg, uchar *a, uint key_length, double *res) +{ + for (; (int)key_length > 0; keyseg += 2) + { + uint32 keyseg_length; + switch ((enum ha_base_keytype) keyseg->type) { + case HA_KEYTYPE_INT8: + RT_D_MBR_KORR(int8, mi_sint1korr, 1, (double)); + break; + case HA_KEYTYPE_BINARY: + RT_D_MBR_KORR(uint8, mi_uint1korr, 1, (double)); + break; + case HA_KEYTYPE_SHORT_INT: + RT_D_MBR_KORR(int16, mi_sint2korr, 2, (double)); + break; + case HA_KEYTYPE_USHORT_INT: + RT_D_MBR_KORR(uint16, mi_uint2korr, 2, (double)); + break; + case HA_KEYTYPE_INT24: + RT_D_MBR_KORR(int32, mi_sint3korr, 3, (double)); + break; + case HA_KEYTYPE_UINT24: + RT_D_MBR_KORR(uint32, mi_uint3korr, 3, (double)); + break; + case HA_KEYTYPE_LONG_INT: + RT_D_MBR_KORR(int32, mi_sint4korr, 4, (double)); + break; + case HA_KEYTYPE_ULONG_INT: + RT_D_MBR_KORR(uint32, mi_uint4korr, 4, (double)); + break; +#ifdef HAVE_LONG_LONG + case HA_KEYTYPE_LONGLONG: + RT_D_MBR_KORR(longlong, mi_sint8korr, 8, (double)); + break; + case HA_KEYTYPE_ULONGLONG: + RT_D_MBR_KORR(longlong, mi_sint8korr, 8, ulonglong2double); + break; +#endif + case HA_KEYTYPE_FLOAT: + RT_D_MBR_GET(float, mi_float4get, 4, (double)); + break; + case HA_KEYTYPE_DOUBLE: + RT_D_MBR_GET(double, mi_float8get, 8, (double)); + break; + case HA_KEYTYPE_END: + key_length = 0; + break; + default: + return 1; + } + keyseg_length= keyseg->length * 2; + key_length-= keyseg_length; + a+= keyseg_length; + } + return 0; +} + +#define RT_COMB_KORR(type, korr_func, store_func, len) \ +{ \ + type amin, amax, bmin, bmax; \ + amin = korr_func(a); \ + bmin = korr_func(b); \ + amax = korr_func(a+len); \ + bmax = korr_func(b+len); \ + amin = MY_MIN(amin, bmin); \ + amax = MY_MAX(amax, bmax); \ + store_func(c, amin); \ + store_func(c+len, amax); \ +} + +#define RT_COMB_GET(type, get_func, store_func, len) \ +{ \ + type amin, amax, bmin, bmax; \ + get_func(amin, a); \ + get_func(bmin, b); \ + get_func(amax, a+len); \ + get_func(bmax, b+len); \ + amin = MY_MIN(amin, bmin); \ + amax = MY_MAX(amax, bmax); \ + store_func(c, amin); \ + store_func(c+len, amax); \ +} + +/* + Creates common minimal bounding rectungle + for two input rectagnles a and b + Result is written to c +*/ + +int rtree_combine_rect(HA_KEYSEG *keyseg, uchar* a, uchar* b, uchar* c, + uint key_length) +{ + for ( ; (int) key_length > 0 ; keyseg += 2) + { + uint32 keyseg_length; + switch ((enum ha_base_keytype) keyseg->type) { + case HA_KEYTYPE_INT8: + RT_COMB_KORR(int8, mi_sint1korr, mi_int1store, 1); + break; + case HA_KEYTYPE_BINARY: + RT_COMB_KORR(uint8, mi_uint1korr, mi_int1store, 1); + break; + case HA_KEYTYPE_SHORT_INT: + RT_COMB_KORR(int16, mi_sint2korr, mi_int2store, 2); + break; + case HA_KEYTYPE_USHORT_INT: + RT_COMB_KORR(uint16, mi_uint2korr, mi_int2store, 2); + break; + case HA_KEYTYPE_INT24: + RT_COMB_KORR(int32, mi_sint3korr, mi_int3store, 3); + break; + case HA_KEYTYPE_UINT24: + RT_COMB_KORR(uint32, mi_uint3korr, mi_int3store, 3); + break; + case HA_KEYTYPE_LONG_INT: + RT_COMB_KORR(int32, mi_sint4korr, mi_int4store, 4); + break; + case HA_KEYTYPE_ULONG_INT: + RT_COMB_KORR(uint32, mi_uint4korr, mi_int4store, 4); + break; +#ifdef HAVE_LONG_LONG + case HA_KEYTYPE_LONGLONG: + RT_COMB_KORR(longlong, mi_sint8korr, mi_int8store, 8); + break; + case HA_KEYTYPE_ULONGLONG: + RT_COMB_KORR(ulonglong, mi_uint8korr, mi_int8store, 8); + break; +#endif + case HA_KEYTYPE_FLOAT: + RT_COMB_GET(float, mi_float4get, mi_float4store, 4); + break; + case HA_KEYTYPE_DOUBLE: + RT_COMB_GET(double, mi_float8get, mi_float8store, 8); + break; + case HA_KEYTYPE_END: + return 0; + default: + return 1; + } + keyseg_length= keyseg->length * 2; + key_length-= keyseg_length; + a+= keyseg_length; + b+= keyseg_length; + c+= keyseg_length; + } + return 0; +} + + +#define RT_OVL_AREA_KORR(type, korr_func, len) \ +{ \ + type amin, amax, bmin, bmax; \ + amin = korr_func(a); \ + bmin = korr_func(b); \ + amax = korr_func(a+len); \ + bmax = korr_func(b+len); \ + amin = MY_MAX(amin, bmin); \ + amax = MY_MIN(amax, bmax); \ + if (amin >= amax) \ + return 0; \ + res *= amax - amin; \ +} + +#define RT_OVL_AREA_GET(type, get_func, len) \ +{ \ + type amin, amax, bmin, bmax; \ + get_func(amin, a); \ + get_func(bmin, b); \ + get_func(amax, a+len); \ + get_func(bmax, b+len); \ + amin = MY_MAX(amin, bmin); \ + amax = MY_MIN(amax, bmax); \ + if (amin >= amax) \ + return 0; \ + res *= amax - amin; \ +} + +/* +Calculates overlapping area of two MBRs a & b +*/ +double rtree_overlapping_area(HA_KEYSEG *keyseg, uchar* a, uchar* b, + uint key_length) +{ + double res = 1; + for (; (int) key_length > 0 ; keyseg += 2) + { + uint32 keyseg_length; + switch ((enum ha_base_keytype) keyseg->type) { + case HA_KEYTYPE_INT8: + RT_OVL_AREA_KORR(int8, mi_sint1korr, 1); + break; + case HA_KEYTYPE_BINARY: + RT_OVL_AREA_KORR(uint8, mi_uint1korr, 1); + break; + case HA_KEYTYPE_SHORT_INT: + RT_OVL_AREA_KORR(int16, mi_sint2korr, 2); + break; + case HA_KEYTYPE_USHORT_INT: + RT_OVL_AREA_KORR(uint16, mi_uint2korr, 2); + break; + case HA_KEYTYPE_INT24: + RT_OVL_AREA_KORR(int32, mi_sint3korr, 3); + break; + case HA_KEYTYPE_UINT24: + RT_OVL_AREA_KORR(uint32, mi_uint3korr, 3); + break; + case HA_KEYTYPE_LONG_INT: + RT_OVL_AREA_KORR(int32, mi_sint4korr, 4); + break; + case HA_KEYTYPE_ULONG_INT: + RT_OVL_AREA_KORR(uint32, mi_uint4korr, 4); + break; +#ifdef HAVE_LONG_LONG + case HA_KEYTYPE_LONGLONG: + RT_OVL_AREA_KORR(longlong, mi_sint8korr, 8); + break; + case HA_KEYTYPE_ULONGLONG: + RT_OVL_AREA_KORR(longlong, mi_sint8korr, 8); + break; +#endif + case HA_KEYTYPE_FLOAT: + RT_OVL_AREA_GET(float, mi_float4get, 4); + break; + case HA_KEYTYPE_DOUBLE: + RT_OVL_AREA_GET(double, mi_float8get, 8); + break; + case HA_KEYTYPE_END: + return res; + default: + return -1; + } + keyseg_length= keyseg->length * 2; + key_length-= keyseg_length; + a+= keyseg_length; + b+= keyseg_length; + } + return res; +} + +#define RT_AREA_INC_KORR(type, korr_func, len) \ +{ \ + type amin, amax, bmin, bmax; \ + amin = korr_func(a); \ + bmin = korr_func(b); \ + amax = korr_func(a+len); \ + bmax = korr_func(b+len); \ + a_area *= (((double)amax) - ((double)amin)); \ + loc_ab_area *= ((double)MY_MAX(amax, bmax) - (double)MY_MIN(amin, bmin)); \ +} + +#define RT_AREA_INC_GET(type, get_func, len)\ +{\ + type amin, amax, bmin, bmax; \ + get_func(amin, a); \ + get_func(bmin, b); \ + get_func(amax, a+len); \ + get_func(bmax, b+len); \ + a_area *= (((double)amax) - ((double)amin)); \ + loc_ab_area *= ((double)MY_MAX(amax, bmax) - (double)MY_MIN(amin, bmin)); \ +} + +/* + Calculates MBR_AREA(a+b) - MBR_AREA(a) + Note: when 'a' and 'b' objects are far from each other, + the area increase can be really big, so this function + can return 'inf' as a result. +*/ +double rtree_area_increase(HA_KEYSEG *keyseg, uchar* a, uchar* b, + uint key_length, double *ab_area) +{ + double a_area= 1.0; + double loc_ab_area= 1.0; + + *ab_area= 1.0; + for (; (int)key_length > 0; keyseg += 2) + { + uint32 keyseg_length; + + if (keyseg->null_bit) /* Handle NULL part */ + return -1; + + switch ((enum ha_base_keytype) keyseg->type) { + case HA_KEYTYPE_INT8: + RT_AREA_INC_KORR(int8, mi_sint1korr, 1); + break; + case HA_KEYTYPE_BINARY: + RT_AREA_INC_KORR(uint8, mi_uint1korr, 1); + break; + case HA_KEYTYPE_SHORT_INT: + RT_AREA_INC_KORR(int16, mi_sint2korr, 2); + break; + case HA_KEYTYPE_USHORT_INT: + RT_AREA_INC_KORR(uint16, mi_uint2korr, 2); + break; + case HA_KEYTYPE_INT24: + RT_AREA_INC_KORR(int32, mi_sint3korr, 3); + break; + case HA_KEYTYPE_UINT24: + RT_AREA_INC_KORR(int32, mi_uint3korr, 3); + break; + case HA_KEYTYPE_LONG_INT: + RT_AREA_INC_KORR(int32, mi_sint4korr, 4); + break; + case HA_KEYTYPE_ULONG_INT: + RT_AREA_INC_KORR(uint32, mi_uint4korr, 4); + break; +#ifdef HAVE_LONG_LONG + case HA_KEYTYPE_LONGLONG: + RT_AREA_INC_KORR(longlong, mi_sint8korr, 8); + break; + case HA_KEYTYPE_ULONGLONG: + RT_AREA_INC_KORR(longlong, mi_sint8korr, 8); + break; +#endif + case HA_KEYTYPE_FLOAT: + RT_AREA_INC_GET(float, mi_float4get, 4); + break; + case HA_KEYTYPE_DOUBLE: + RT_AREA_INC_GET(double, mi_float8get, 8); + break; + case HA_KEYTYPE_END: + goto safe_end; + default: + return -1; + } + keyseg_length= keyseg->length * 2; + key_length-= keyseg_length; + a+= keyseg_length; + b+= keyseg_length; + } +safe_end: + *ab_area= loc_ab_area; + return loc_ab_area - a_area; +} + +#define RT_PERIM_INC_KORR(type, korr_func, len) \ +{ \ + type amin, amax, bmin, bmax; \ + amin = korr_func(a); \ + bmin = korr_func(b); \ + amax = korr_func(a+len); \ + bmax = korr_func(b+len); \ + a_perim+= (((double)amax) - ((double)amin)); \ + *ab_perim+= ((double)MY_MAX(amax, bmax) - (double)MY_MIN(amin, bmin)); \ +} + +#define RT_PERIM_INC_GET(type, get_func, len)\ +{\ + type amin, amax, bmin, bmax; \ + get_func(amin, a); \ + get_func(bmin, b); \ + get_func(amax, a+len); \ + get_func(bmax, b+len); \ + a_perim+= (((double)amax) - ((double)amin)); \ + *ab_perim+= ((double)MY_MAX(amax, bmax) - (double)MY_MIN(amin, bmin)); \ +} + +/* +Calculates MBR_PERIMETER(a+b) - MBR_PERIMETER(a) +*/ +double rtree_perimeter_increase(HA_KEYSEG *keyseg, uchar* a, uchar* b, + uint key_length, double *ab_perim) +{ + double a_perim = 0.0; + + *ab_perim= 0.0; + for (; (int)key_length > 0; keyseg += 2) + { + uint32 keyseg_length; + + if (keyseg->null_bit) /* Handle NULL part */ + return -1; + + switch ((enum ha_base_keytype) keyseg->type) { + case HA_KEYTYPE_INT8: + RT_PERIM_INC_KORR(int8, mi_sint1korr, 1); + break; + case HA_KEYTYPE_BINARY: + RT_PERIM_INC_KORR(uint8, mi_uint1korr, 1); + break; + case HA_KEYTYPE_SHORT_INT: + RT_PERIM_INC_KORR(int16, mi_sint2korr, 2); + break; + case HA_KEYTYPE_USHORT_INT: + RT_PERIM_INC_KORR(uint16, mi_uint2korr, 2); + break; + case HA_KEYTYPE_INT24: + RT_PERIM_INC_KORR(int32, mi_sint3korr, 3); + break; + case HA_KEYTYPE_UINT24: + RT_PERIM_INC_KORR(int32, mi_uint3korr, 3); + break; + case HA_KEYTYPE_LONG_INT: + RT_PERIM_INC_KORR(int32, mi_sint4korr, 4); + break; + case HA_KEYTYPE_ULONG_INT: + RT_PERIM_INC_KORR(uint32, mi_uint4korr, 4); + break; +#ifdef HAVE_LONG_LONG + case HA_KEYTYPE_LONGLONG: + RT_PERIM_INC_KORR(longlong, mi_sint8korr, 8); + break; + case HA_KEYTYPE_ULONGLONG: + RT_PERIM_INC_KORR(longlong, mi_sint8korr, 8); + break; +#endif + case HA_KEYTYPE_FLOAT: + RT_PERIM_INC_GET(float, mi_float4get, 4); + break; + case HA_KEYTYPE_DOUBLE: + RT_PERIM_INC_GET(double, mi_float8get, 8); + break; + case HA_KEYTYPE_END: + return *ab_perim - a_perim; + default: + return -1; + } + keyseg_length= keyseg->length * 2; + key_length-= keyseg_length; + a+= keyseg_length; + b+= keyseg_length; + } + return *ab_perim - a_perim; +} + + +#define RT_PAGE_MBR_KORR(type, korr_func, store_func, len) \ +{ \ + type amin, amax, bmin, bmax; \ + amin = korr_func(k + inc); \ + amax = korr_func(k + inc + len); \ + k = rt_PAGE_NEXT_KEY(k, k_len, nod_flag); \ + for (; k < last; k = rt_PAGE_NEXT_KEY(k, k_len, nod_flag)) \ +{ \ + bmin = korr_func(k + inc); \ + bmax = korr_func(k + inc + len); \ + if (amin > bmin) \ + amin = bmin; \ + if (amax < bmax) \ + amax = bmax; \ +} \ + store_func(c, amin); \ + c += len; \ + store_func(c, amax); \ + c += len; \ + inc += 2 * len; \ +} + +#define RT_PAGE_MBR_GET(type, get_func, store_func, len) \ +{ \ + type amin, amax, bmin, bmax; \ + get_func(amin, k + inc); \ + get_func(amax, k + inc + len); \ + k = rt_PAGE_NEXT_KEY(k, k_len, nod_flag); \ + for (; k < last; k = rt_PAGE_NEXT_KEY(k, k_len, nod_flag)) \ +{ \ + get_func(bmin, k + inc); \ + get_func(bmax, k + inc + len); \ + if (amin > bmin) \ + amin = bmin; \ + if (amax < bmax) \ + amax = bmax; \ +} \ + store_func(c, amin); \ + c += len; \ + store_func(c, amax); \ + c += len; \ + inc += 2 * len; \ +} + +/* +Calculates key page total MBR = MBR(key1) + MBR(key2) + ... +*/ +int rtree_page_mbr(MI_INFO *info, HA_KEYSEG *keyseg, uchar *page_buf, + uchar *c, uint key_length) +{ + uint inc = 0; + uint k_len = key_length; + uint nod_flag = mi_test_if_nod(page_buf); + uchar *k; + uchar *last = rt_PAGE_END(page_buf); + + for (; (int)key_length > 0; keyseg += 2) + { + key_length -= keyseg->length * 2; + + /* Handle NULL part */ + if (keyseg->null_bit) + { + return 1; + } + + k = rt_PAGE_FIRST_KEY(page_buf, nod_flag); + + switch ((enum ha_base_keytype) keyseg->type) { + case HA_KEYTYPE_INT8: + RT_PAGE_MBR_KORR(int8, mi_sint1korr, mi_int1store, 1); + break; + case HA_KEYTYPE_BINARY: + RT_PAGE_MBR_KORR(uint8, mi_uint1korr, mi_int1store, 1); + break; + case HA_KEYTYPE_SHORT_INT: + RT_PAGE_MBR_KORR(int16, mi_sint2korr, mi_int2store, 2); + break; + case HA_KEYTYPE_USHORT_INT: + RT_PAGE_MBR_KORR(uint16, mi_uint2korr, mi_int2store, 2); + break; + case HA_KEYTYPE_INT24: + RT_PAGE_MBR_KORR(int32, mi_sint3korr, mi_int3store, 3); + break; + case HA_KEYTYPE_UINT24: + RT_PAGE_MBR_KORR(uint32, mi_uint3korr, mi_int3store, 3); + break; + case HA_KEYTYPE_LONG_INT: + RT_PAGE_MBR_KORR(int32, mi_sint4korr, mi_int4store, 4); + break; + case HA_KEYTYPE_ULONG_INT: + RT_PAGE_MBR_KORR(uint32, mi_uint4korr, mi_int4store, 4); + break; +#ifdef HAVE_LONG_LONG + case HA_KEYTYPE_LONGLONG: + RT_PAGE_MBR_KORR(longlong, mi_sint8korr, mi_int8store, 8); + break; + case HA_KEYTYPE_ULONGLONG: + RT_PAGE_MBR_KORR(ulonglong, mi_uint8korr, mi_int8store, 8); + break; +#endif + case HA_KEYTYPE_FLOAT: + RT_PAGE_MBR_GET(float, mi_float4get, mi_float4store, 4); + break; + case HA_KEYTYPE_DOUBLE: + RT_PAGE_MBR_GET(double, mi_float8get, mi_float8store, 8); + break; + case HA_KEYTYPE_END: + return 0; + default: + return 1; + } + } + return 0; +} + +#endif /*HAVE_RTREE_KEYS*/ diff --git a/storage/myisam/rt_mbr.h b/storage/myisam/rt_mbr.h new file mode 100644 index 00000000..43e55d39 --- /dev/null +++ b/storage/myisam/rt_mbr.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2002, 2004-2006 MySQL AB + Use is subject to license terms + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +#ifndef _rt_mbr_h +#define _rt_mbr_h + +#ifdef HAVE_RTREE_KEYS + +int rtree_key_cmp(HA_KEYSEG *keyseg, uchar *a, uchar *b, uint key_length, + uint nextflag); +int rtree_combine_rect(HA_KEYSEG *keyseg,uchar *, uchar *, uchar*, + uint key_length); +double rtree_rect_volume(HA_KEYSEG *keyseg, uchar*, uint key_length); +int rtree_d_mbr(HA_KEYSEG *keyseg, uchar *a, uint key_length, double *res); +double rtree_overlapping_area(HA_KEYSEG *keyseg, uchar *a, uchar *b, + uint key_length); +double rtree_area_increase(HA_KEYSEG *keyseg, uchar *a, uchar *b, + uint key_length, double *ab_area); +double rtree_perimeter_increase(HA_KEYSEG *keyseg, uchar* a, uchar* b, + uint key_length, double *ab_perim); +int rtree_page_mbr(MI_INFO *info, HA_KEYSEG *keyseg, uchar *page_buf, + uchar* c, uint key_length); +#endif /*HAVE_RTREE_KEYS*/ +#endif /* _rt_mbr_h */ diff --git a/storage/myisam/rt_split.c b/storage/myisam/rt_split.c new file mode 100644 index 00000000..1adcdb8e --- /dev/null +++ b/storage/myisam/rt_split.c @@ -0,0 +1,348 @@ +/* + Copyright (c) 2002, 2015, Oracle and/or its affiliates + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "myisamdef.h" + +#ifdef HAVE_RTREE_KEYS + +#include "rt_index.h" +#include "rt_key.h" +#include "rt_mbr.h" + +typedef struct +{ + double square; + int n_node; + uchar *key; + double *coords; +} SplitStruct; + +inline static double *reserve_coords(double **d_buffer, int n_dim) +{ + double *coords = *d_buffer; + (*d_buffer) += n_dim * 2; + return coords; +} + +static void mbr_join(double *a, const double *b, int n_dim) +{ + double *end = a + n_dim * 2; + do + { + if (a[0] > b[0]) + a[0] = b[0]; + + if (a[1] < b[1]) + a[1] = b[1]; + + a += 2; + b += 2; + }while (a != end); +} + +/* +Counts the square of mbr which is a join of a and b +*/ +static double mbr_join_square(const double *a, const double *b, int n_dim) +{ + const double *end = a + n_dim * 2; + double square = 1.0; + do + { + square *= + ((a[1] < b[1]) ? b[1] : a[1]) - ((a[0] > b[0]) ? b[0] : a[0]); + + a += 2; + b += 2; + }while (a != end); + + /* Check if not finite (i.e. infinity or NaN) */ + if (!isfinite(square)) + square = DBL_MAX; + + return square; +} + +static double count_square(const double *a, int n_dim) +{ + const double *end = a + n_dim * 2; + double square = 1.0; + do + { + square *= a[1] - a[0]; + a += 2; + }while (a != end); + return square; +} + +inline static void copy_coords(double *dst, const double *src, int n_dim) +{ + memcpy(dst, src, sizeof(double) * (n_dim * 2)); +} + +/* +Select two nodes to collect group upon +*/ +static void pick_seeds(SplitStruct *node, int n_entries, + SplitStruct **seed_a, SplitStruct **seed_b, int n_dim) +{ + SplitStruct *cur1; + SplitStruct *lim1 = node + (n_entries - 1); + SplitStruct *cur2; + SplitStruct *lim2 = node + n_entries; + + double max_d = -DBL_MAX; + double d; + + *seed_a = node; + *seed_b = node + 1; + + for (cur1 = node; cur1 < lim1; ++cur1) + { + for (cur2=cur1 + 1; cur2 < lim2; ++cur2) + { + + d = mbr_join_square(cur1->coords, cur2->coords, n_dim) - cur1->square - + cur2->square; + if (d > max_d) + { + max_d = d; + *seed_a = cur1; + *seed_b = cur2; + } + } + } +} + +/* +Select next node and group where to add +*/ +static void pick_next(SplitStruct *node, int n_entries, double *g1, double *g2, + SplitStruct **choice, int *n_group, int n_dim) +{ + SplitStruct *cur = node; + SplitStruct *end = node + n_entries; + + double max_diff = -DBL_MAX; + + for (; cur<end; ++cur) + { + double diff; + double abs_diff; + + if (cur->n_node) + { + continue; + } + + diff = mbr_join_square(g1, cur->coords, n_dim) - + mbr_join_square(g2, cur->coords, n_dim); + + abs_diff = fabs(diff); + if (abs_diff > max_diff) + { + max_diff = abs_diff; + *n_group = 1 + (diff > 0); + *choice = cur; + } + } +} + +/* +Mark not-in-group entries as n_group +*/ +static void mark_all_entries(SplitStruct *node, int n_entries, int n_group) +{ + SplitStruct *cur = node; + SplitStruct *end = node + n_entries; + for (; cur<end; ++cur) + { + if (cur->n_node) + { + continue; + } + cur->n_node = n_group; + } +} + +static int split_rtree_node(SplitStruct *node, int n_entries, + int all_size, /* Total key's size */ + int key_size, + int min_size, /* Minimal group size */ + int size1, int size2 /* initial group sizes */, + double **d_buffer, int n_dim) +{ + SplitStruct *cur; + SplitStruct *UNINIT_VAR(a), *UNINIT_VAR(b); + double *g1 = reserve_coords(d_buffer, n_dim); + double *g2 = reserve_coords(d_buffer, n_dim); + SplitStruct *UNINIT_VAR(next); + int UNINIT_VAR(next_node); + int i; + SplitStruct *end = node + n_entries; + + if (all_size < min_size * 2) + { + return 1; + } + + cur = node; + for (; cur<end; ++cur) + { + cur->square = count_square(cur->coords, n_dim); + cur->n_node = 0; + } + + pick_seeds(node, n_entries, &a, &b, n_dim); + a->n_node = 1; + b->n_node = 2; + + + copy_coords(g1, a->coords, n_dim); + size1 += key_size; + copy_coords(g2, b->coords, n_dim); + size2 += key_size; + + + for (i=n_entries - 2; i>0; --i) + { + if (all_size - (size2 + key_size) < min_size) /* Can't write into group 2 */ + { + mark_all_entries(node, n_entries, 1); + break; + } + + if (all_size - (size1 + key_size) < min_size) /* Can't write into group 1 */ + { + mark_all_entries(node, n_entries, 2); + break; + } + + pick_next(node, n_entries, g1, g2, &next, &next_node, n_dim); + if (next_node == 1) + { + size1 += key_size; + mbr_join(g1, next->coords, n_dim); + } + else + { + size2 += key_size; + mbr_join(g2, next->coords, n_dim); + } + next->n_node = next_node; + } + + return 0; +} + +int rtree_split_page(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *page, uchar *key, + uint key_length, my_off_t *new_page_offs) +{ + int n1, n2; /* Number of items in groups */ + + SplitStruct *task; + SplitStruct *cur; + SplitStruct *stop; + double *coord_buf; + double *next_coord; + int n_dim; + uchar *source_cur, *cur1, *cur2; + uchar *new_page= info->buff; + int err_code= 0; + uint nod_flag= mi_test_if_nod(page); + uint full_length= key_length + (nod_flag ? nod_flag : + info->s->base.rec_reflength); + int max_keys= (mi_getint(page)-2) / (full_length); + DBUG_ENTER("rtree_split_page"); + DBUG_PRINT("rtree", ("splitting block")); + + n_dim = keyinfo->keysegs / 2; + + if (!(coord_buf= (double*) my_alloca(n_dim * 2 * sizeof(double) * + (max_keys + 1 + 4) + + sizeof(SplitStruct) * (max_keys + 1)))) + DBUG_RETURN(-1); /* purecov: inspected */ + + task= (SplitStruct *)(coord_buf + n_dim * 2 * (max_keys + 1 + 4)); + + next_coord = coord_buf; + + stop = task + max_keys; + source_cur = rt_PAGE_FIRST_KEY(page, nod_flag); + + for (cur = task; cur < stop; ++cur, source_cur = rt_PAGE_NEXT_KEY(source_cur, + key_length, nod_flag)) + { + cur->coords = reserve_coords(&next_coord, n_dim); + cur->key = source_cur; + rtree_d_mbr(keyinfo->seg, source_cur, key_length, cur->coords); + } + + cur->coords = reserve_coords(&next_coord, n_dim); + rtree_d_mbr(keyinfo->seg, key, key_length, cur->coords); + cur->key = key; + + if (split_rtree_node(task, max_keys + 1, + mi_getint(page) + full_length + 2, full_length, + rt_PAGE_MIN_SIZE(keyinfo->block_length), + 2, 2, &next_coord, n_dim)) + { + err_code = 1; + goto split_err; + } + + info->buff_used= 1; + stop = task + (max_keys + 1); + cur1 = rt_PAGE_FIRST_KEY(page, nod_flag); + cur2 = rt_PAGE_FIRST_KEY(new_page, nod_flag); + + n1= n2 = 0; + for (cur = task; cur < stop; ++cur) + { + uchar *to; + if (cur->n_node == 1) + { + to = cur1; + cur1 = rt_PAGE_NEXT_KEY(cur1, key_length, nod_flag); + ++n1; + } + else + { + to = cur2; + cur2 = rt_PAGE_NEXT_KEY(cur2, key_length, nod_flag); + ++n2; + } + if (to != cur->key) + memcpy(to - nod_flag, cur->key - nod_flag, full_length); + } + + mi_putint(page, 2 + n1 * full_length, nod_flag); + mi_putint(new_page, 2 + n2 * full_length, nod_flag); + + if ((*new_page_offs= _mi_new(info, keyinfo, DFLT_INIT_HITS)) == + HA_OFFSET_ERROR) + err_code= -1; + else + err_code= _mi_write_keypage(info, keyinfo, *new_page_offs, + DFLT_INIT_HITS, new_page); + DBUG_PRINT("rtree", ("split new block: %lu", (ulong) *new_page_offs)); + +split_err: + my_afree((uchar*) coord_buf); + DBUG_RETURN(err_code); +} + +#endif /*HAVE_RTREE_KEYS*/ diff --git a/storage/myisam/rt_test.c b/storage/myisam/rt_test.c new file mode 100644 index 00000000..a35d41a0 --- /dev/null +++ b/storage/myisam/rt_test.c @@ -0,0 +1,439 @@ +/* Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Testing of the basic functions of a MyISAM rtree table */ +/* Written by Alex Barkov who has a shared copyright to this code */ + + +#include <my_global.h> +#include "myisam.h" + +#ifdef HAVE_RTREE_KEYS + +#include "rt_index.h" + +#define MAX_REC_LENGTH 1024 +#define ndims 2 +#define KEYALG HA_KEY_ALG_RTREE + +static int read_with_pos(MI_INFO * file, int silent); +static void create_record(uchar *record,uint rownr); +static void create_record1(uchar *record,uint rownr); +static void print_record(uchar * record,my_off_t offs,const char * tail); +static int run_test(const char *filename); + +static double rt_data[]= +{ + /*1*/ 0,10,0,10, + /*2*/ 5,15,0,10, + /*3*/ 0,10,5,15, + /*4*/ 10,20,10,20, + /*5*/ 0,10,0,10, + /*6*/ 5,15,0,10, + /*7*/ 0,10,5,15, + /*8*/ 10,20,10,20, + /*9*/ 0,10,0,10, + /*10*/ 5,15,0,10, + /*11*/ 0,10,5,15, + /*12*/ 10,20,10,20, + /*13*/ 0,10,0,10, + /*14*/ 5,15,0,10, + /*15*/ 0,10,5,15, + /*16*/ 10,20,10,20, + /*17*/ 5,15,0,10, + /*18*/ 0,10,5,15, + /*19*/ 10,20,10,20, + /*20*/ 0,10,0,10, + + /*1*/ 100,110,0,10, + /*2*/ 105,115,0,10, + /*3*/ 100,110,5,15, + /*4*/ 110,120,10,20, + /*5*/ 100,110,0,10, + /*6*/ 105,115,0,10, + /*7*/ 100,110,5,15, + /*8*/ 110,120,10,20, + /*9*/ 100,110,0,10, + /*10*/ 105,115,0,10, + /*11*/ 100,110,5,15, + /*12*/ 110,120,10,20, + /*13*/ 100,110,0,10, + /*14*/ 105,115,0,10, + /*15*/ 100,110,5,15, + /*16*/ 110,120,10,20, + /*17*/ 105,115,0,10, + /*18*/ 100,110,5,15, + /*19*/ 110,120,10,20, + /*20*/ 100,110,0,10, + -1 +}; + +int main(int argc __attribute__((unused)),char *argv[] __attribute__((unused))) +{ + MY_INIT(argv[0]); + exit(run_test("rt_test")); +} + + +static int run_test(const char *filename) +{ + MI_INFO *file; + MI_UNIQUEDEF uniquedef; + MI_CREATE_INFO create_info; + MI_COLUMNDEF recinfo[20]; + MI_KEYDEF keyinfo[20]; + HA_KEYSEG keyseg[20]; + key_range range; + + int silent=0; + int opt_unique=0; + int create_flag=0; + int key_type=HA_KEYTYPE_DOUBLE; + int key_length=8; + int null_fields=0; + int nrecords=sizeof(rt_data)/(sizeof(double)*4);/* 3000;*/ + int uniques=0; + int i, max_i; + int error; + int row_count=0; + uchar record[MAX_REC_LENGTH]; + uchar read_record[MAX_REC_LENGTH]; + int upd= 10; + ha_rows hrows; + page_range pages; + + bzero(&uniquedef, sizeof(uniquedef)); + bzero(&create_info, sizeof(create_info)); + bzero(recinfo, sizeof(recinfo)); + bzero(keyinfo, sizeof(keyinfo)); + bzero(keyseg, sizeof(keyseg)); + + /* Define a column for NULLs and DEL markers*/ + + recinfo[0].type=FIELD_NORMAL; + recinfo[0].length=1; /* For NULL bits */ + + /* Define 2*ndims columns for coordinates*/ + + for (i=1; i<=2*ndims ;i++){ + recinfo[i].type=FIELD_NORMAL; + recinfo[i].length=key_length; + } + + /* Define a key with 2*ndims segments */ + + keyinfo[0].seg=keyseg; + keyinfo[0].keysegs=2*ndims; + keyinfo[0].flag=0; + keyinfo[0].key_alg=KEYALG; + + for (i=0; i<2*ndims; i++){ + keyinfo[0].seg[i].type= key_type; + keyinfo[0].seg[i].flag=0; /* Things like HA_REVERSE_SORT */ + keyinfo[0].seg[i].start= (key_length*i)+1; + keyinfo[0].seg[i].length=key_length; + keyinfo[0].seg[i].null_bit= null_fields ? 2 : 0; + keyinfo[0].seg[i].null_pos=0; + keyinfo[0].seg[i].language=default_charset_info->number; + } + + if (!silent) + printf("- Creating isam-file\n"); + + create_info.max_rows=10000000; + + if (mi_create(filename, + 1, /* keys */ + keyinfo, + 1+2*ndims+opt_unique, /* columns */ + recinfo,uniques,&uniquedef,&create_info,create_flag)) + goto err; + + if (!silent) + printf("- Open isam-file\n"); + + if (!(file=mi_open(filename,2,HA_OPEN_ABORT_IF_LOCKED))) + goto err; + + if (!silent) + printf("- Writing key:s\n"); + + for (i=0; i<nrecords; i++ ) + { + create_record(record,i); + error=mi_write(file,record); + print_record(record,mi_position(file),"\n"); + if (!error) + { + row_count++; + } + else + { + printf("mi_write: %d\n", error); + goto err; + } + } + + if ((error=read_with_pos(file,silent))) + goto err; + + if (!silent) + printf("- Reading rows with key\n"); + + for (i=0 ; i < nrecords ; i++) + { + my_errno=0; + create_record(record,i); + + bzero((char*) read_record,MAX_REC_LENGTH); + error=mi_rkey(file,read_record,0,record+1,HA_WHOLE_KEY,HA_READ_MBR_EQUAL); + + if (error && error!=HA_ERR_KEY_NOT_FOUND) + { + printf(" mi_rkey: %3d errno: %3d\n",error,my_errno); + goto err; + } + if (error == HA_ERR_KEY_NOT_FOUND) + { + print_record(record,mi_position(file)," NOT FOUND\n"); + continue; + } + print_record(read_record,mi_position(file),"\n"); + } + + if (!silent) + printf("- Deleting rows\n"); + for (i=0; i < nrecords/4; i++) + { + my_errno=0; + bzero((char*) read_record,MAX_REC_LENGTH); + error=mi_rrnd(file,read_record,i == 0 ? 0L : HA_OFFSET_ERROR); + if (error) + { + printf("pos: %2d mi_rrnd: %3d errno: %3d\n",i,error,my_errno); + goto err; + } + print_record(read_record,mi_position(file),"\n"); + + error=mi_delete(file,read_record); + if (error) + { + printf("pos: %2d mi_delete: %3d errno: %3d\n",i,error,my_errno); + goto err; + } + } + + if (!silent) + printf("- Updating rows with position\n"); + /* We are looking for nrecords-necords/2 non-deleted records */ + for (i=0, max_i= nrecords - nrecords/2; i < max_i ; i++) + { + my_errno=0; + bzero((char*) read_record,MAX_REC_LENGTH); + error=mi_rrnd(file,read_record,i == 0 ? 0L : HA_OFFSET_ERROR); + if (error) + { + if (error==HA_ERR_RECORD_DELETED) + { + printf("found deleted record\n"); + max_i++; /* don't count such record */ + continue; + } + printf("pos: %2d mi_rrnd: %3d errno: %3d\n",i,error,my_errno); + goto err; + } + print_record(read_record,mi_position(file),""); + create_record(record,i+nrecords*upd); + printf("\t-> "); + print_record(record,mi_position(file),"\n"); + error=mi_update(file,read_record,record); + if (error) + { + printf("pos: %2d mi_update: %3d errno: %3d\n",i,error,my_errno); + goto err; + } + } + + if ((error=read_with_pos(file,silent))) + goto err; + + if (!silent) + printf("- Test mi_rkey then a sequence of mi_rnext_same\n"); + + create_record(record, nrecords*4/5); + print_record(record,0," search for\n"); + + if ((error=mi_rkey(file,read_record,0,record+1,HA_WHOLE_KEY, + HA_READ_MBR_INTERSECT))) + { + printf("mi_rkey: %3d errno: %3d\n",error,my_errno); + goto err; + } + print_record(read_record,mi_position(file)," mi_rkey\n"); + row_count=1; + + for (;;) + { + if ((error=mi_rnext_same(file,read_record))) + { + if (error==HA_ERR_END_OF_FILE) + break; + printf("mi_next: %3d errno: %3d\n",error,my_errno); + goto err; + } + print_record(read_record,mi_position(file)," mi_rnext_same\n"); + row_count++; + } + printf(" %d rows\n",row_count); + + if (!silent) + printf("- Test mi_rfirst then a sequence of mi_rnext\n"); + + error=mi_rfirst(file,read_record,0); + if (error) + { + printf("mi_rfirst: %3d errno: %3d\n",error,my_errno); + goto err; + } + row_count=1; + print_record(read_record,mi_position(file)," mi_frirst\n"); + + for (i=0;i<nrecords;i++) + { + if ((error=mi_rnext(file,read_record,0))) + { + if (error==HA_ERR_END_OF_FILE) + break; + printf("mi_next: %3d errno: %3d\n",error,my_errno); + goto err; + } + print_record(read_record,mi_position(file)," mi_rnext\n"); + row_count++; + } + printf(" %d rows\n",row_count); + + if (!silent) + printf("- Test mi_records_in_range()\n"); + + create_record1(record, nrecords*4/5); + print_record(record,0,"\n"); + + range.key= record+1; + range.length= 1000; /* Big enough */ + range.flag= HA_READ_MBR_INTERSECT; + hrows= mi_records_in_range(file, 0, &range, (key_range*) 0, &pages); + printf(" %ld rows\n", (long) hrows); + + if (mi_close(file)) goto err; + my_end(MY_CHECK_ERROR); + + return 0; + +err: + printf("got error: %3d when using myisam-database\n",my_errno); + return 1; /* skip warning */ +} + + + +static int read_with_pos (MI_INFO * file,int silent) +{ + int error; + int i; + uchar read_record[MAX_REC_LENGTH]; + + if (!silent) + printf("- Reading rows with position\n"); + for (i=0;;i++) + { + my_errno=0; + bzero((char*) read_record,MAX_REC_LENGTH); + error=mi_rrnd(file,read_record,i == 0 ? 0L : HA_OFFSET_ERROR); + if (error) + { + if (error==HA_ERR_END_OF_FILE) + break; + if (error==HA_ERR_RECORD_DELETED) + continue; + printf("pos: %2d mi_rrnd: %3d errno: %3d\n",i,error,my_errno); + return error; + } + print_record(read_record,mi_position(file),"\n"); + } + return 0; +} + + +static void print_record(uchar * record, + my_off_t offs __attribute__((unused)), + const char * tail) +{ + int i; + uchar * pos; + double c; + + printf(" rec=(%d)",(unsigned char)record[0]); + for ( pos=record+1, i=0; i<2*ndims; i++) + { + memcpy(&c,pos,sizeof(c)); + float8get(c,pos); + printf(" %.14g ",c); + pos+=sizeof(c); + } + printf("pos=%ld",(long int)offs); + printf("%s",tail); +} + + + +static void create_record1(uchar *record,uint rownr) +{ + int i; + uchar * pos; + double c=rownr+10; + + bzero((char*) record,MAX_REC_LENGTH); + record[0]=0x01; /* DEL marker */ + + for (pos=record+1, i=0; i<2*ndims; i++) + { + memcpy(pos,&c,sizeof(c)); + float8store(pos,c); + pos+=sizeof(c); + } +} + + +static void create_record(uchar *record,uint rownr) +{ + int i; + uchar *pos; + double *data= rt_data+rownr*4; + record[0]=0x01; /* DEL marker */ + for (pos=record+1, i=0; i<ndims*2; i++) + { + float8store(pos,data[i]); + pos+=8; + } +} + +#else +int main(int argc __attribute__((unused)),char *argv[] __attribute__((unused))) +{ + exit(0); +} +#endif /*HAVE_RTREE_KEYS*/ + +#include "mi_extrafunc.h" diff --git a/storage/myisam/sort.c b/storage/myisam/sort.c new file mode 100644 index 00000000..375c1840 --- /dev/null +++ b/storage/myisam/sort.c @@ -0,0 +1,1146 @@ +/* Copyright (c) 2000, 2012, Oracle and/or its affiliates. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* + Creates a index for a database by reading keys, sorting them and outputing + them in sorted order through MI_SORT_INFO functions. +*/ + +#include "fulltext.h" +#if defined(_WIN32) +#include <fcntl.h> +#else +#include <stddef.h> +#endif +#include <queues.h> + +/* static variables */ + +#undef DISK_BUFFER_SIZE + +#define MERGEBUFF 15 +#define MERGEBUFF2 31 +#define DISK_BUFFER_SIZE (IO_SIZE*128) + +/* How many keys we can keep in memory */ +typedef ulonglong ha_keys; + +/* + Pointers of functions for store and read keys from temp file +*/ + +extern void print_error(const char *fmt,...); + +/* Functions defined in this file */ + +static ha_rows find_all_keys(MI_SORT_PARAM *info, ha_keys keys, + uchar **sort_keys, + DYNAMIC_ARRAY *buffpek, size_t *maxbuffer, + IO_CACHE *tempfile, + IO_CACHE *tempfile_for_exceptions); +static int write_keys(MI_SORT_PARAM *info,uchar **sort_keys, + ha_keys count, BUFFPEK *buffpek,IO_CACHE *tempfile); +static int write_key(MI_SORT_PARAM *info, uchar *key, + IO_CACHE *tempfile); +static int write_index(MI_SORT_PARAM *info,uchar * *sort_keys, + ha_keys count); +static int merge_many_buff(MI_SORT_PARAM *info, ha_keys keys, + uchar * *sort_keys, + BUFFPEK *buffpek, size_t *maxbuffer, + IO_CACHE *t_file); +static my_off_t read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek, + uint sort_length); +static int merge_buffers(MI_SORT_PARAM *info, ha_keys keys, + IO_CACHE *from_file, IO_CACHE *to_file, + uchar * *sort_keys, BUFFPEK *lastbuff, + BUFFPEK *Fb, BUFFPEK *Tb); +static int merge_index(MI_SORT_PARAM *,ha_keys,uchar **,BUFFPEK *, size_t, + IO_CACHE *); +static int flush_ft_buf(MI_SORT_PARAM *info); + +static int write_keys_varlen(MI_SORT_PARAM *info,uchar **sort_keys, + ha_keys count, BUFFPEK *buffpek, + IO_CACHE *tempfile); +static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek, + uint sort_length); +static int write_merge_key(MI_SORT_PARAM *info, IO_CACHE *to_file, + uchar *key, uint sort_length, ha_keys count); +static int write_merge_key_varlen(MI_SORT_PARAM *info, + IO_CACHE *to_file, + uchar* key, uint sort_length, + ha_keys count); +static inline int +my_var_write(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs); + + +/* + Sets the appropriate read and write methods for the MI_SORT_PARAM + based on the variable length key flag. +*/ +static void set_sort_param_read_write(MI_SORT_PARAM *sort_param) +{ + if (sort_param->keyinfo->flag & HA_VAR_LENGTH_KEY) + { + sort_param->write_keys= write_keys_varlen; + sort_param->read_to_buffer= read_to_buffer_varlen; + sort_param->write_key= write_merge_key_varlen; + } + else + { + sort_param->write_keys= write_keys; + sort_param->read_to_buffer= read_to_buffer; + sort_param->write_key= write_merge_key; + } +} + + +/* + Creates a index of sorted keys + + SYNOPSIS + _create_index_by_sort() + info Sort parameters + no_messages Set to 1 if no output + sortbuff_size Size if sortbuffer to allocate + + RESULT + 0 ok + <> 0 Error +*/ + +int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, + ulonglong sortbuff_size) +{ + int error; + uint sort_length; + size_t maxbuffer; + ulonglong memavl, old_memavl; + DYNAMIC_ARRAY buffpek; + ha_rows records, UNINIT_VAR(keys); + uchar **sort_keys; + IO_CACHE tempfile, tempfile_for_exceptions; + DBUG_ENTER("_create_index_by_sort"); + DBUG_PRINT("enter",("sort_length: %u", info->key_length)); + + set_sort_param_read_write(info); + + my_b_clear(&tempfile); + my_b_clear(&tempfile_for_exceptions); + bzero((char*) &buffpek,sizeof(buffpek)); + sort_keys= (uchar **) NULL; error= 1; + maxbuffer=1; + + memavl= MY_MAX(sortbuff_size, MIN_SORT_BUFFER); + records= info->sort_info->max_records; + sort_length= info->key_length; + + while (memavl >= MIN_SORT_BUFFER) + { + /* Check if we can fit all keys into memory */ + if (((ulonglong) (records + 1) * + (sort_length + sizeof(char*)) <= memavl)) + keys= records+1; + else if ((info->sort_info->param->testflag & + (T_FORCE_SORT_MEMORY | T_CREATE_MISSING_KEYS)) == + T_FORCE_SORT_MEMORY) + { + /* + Use all of the given sort buffer for key data. + Allocate 1000 buffers at a start for new data. More buffers + will be allocated when needed. + */ + keys= memavl / (sort_length+sizeof(char*)); + maxbuffer= (size_t) MY_MIN((ulonglong) 1000, (records / keys)+1); + } + else + { + /* + All keys can't fit in memory. + Calculate how many keys + buffers we can keep in memory + */ + size_t maxbuffer_org; + do + { + maxbuffer_org= maxbuffer; + if (memavl < sizeof(BUFFPEK) * maxbuffer || + (keys= (memavl-sizeof(BUFFPEK)*maxbuffer)/ + (sort_length+sizeof(char*))) <= 1) + { + mi_check_print_error(info->sort_info->param, + "myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu rows: %llu sort_length: %u", + sortbuff_size, (ulonglong) records, + sort_length); + my_errno= ENOMEM; + goto err; + } + if (keys < maxbuffer) + { + /* + There must be sufficient memory for at least one key per BUFFPEK, + otherwise repair by sort/parallel repair cannot operate. + */ + keys= maxbuffer; + break; + } + } + while ((maxbuffer= (size_t) (records/(keys-1)+1)) != maxbuffer_org); + } + + if ((sort_keys= ((uchar **) + my_malloc(PSI_INSTRUMENT_ME, + (size_t) (keys*(sort_length+sizeof(char*))+ + HA_FT_MAXBYTELEN), MYF(0))))) + { + if (my_init_dynamic_array(PSI_INSTRUMENT_ME, &buffpek, sizeof(BUFFPEK), + maxbuffer, MY_MIN(maxbuffer/2, 1000), MYF(0))) + { + my_free(sort_keys); + sort_keys= 0; + } + else + break; + } + old_memavl=memavl; + if ((memavl= memavl/4*3) < MIN_SORT_BUFFER && old_memavl > MIN_SORT_BUFFER) + memavl= MIN_SORT_BUFFER; + } + if (memavl < MIN_SORT_BUFFER) + { + /* purecov: begin inspected */ + mi_check_print_error(info->sort_info->param, + "myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu rows: %llu sort_length: %u", + sortbuff_size, (ulonglong) records, sort_length); + my_errno= ENOMEM; + goto err; + /* purecov: end inspected */ + } + (*info->lock_in_memory)(info->sort_info->param);/* Everything is allocated */ + + if (!no_messages) + my_fprintf(stdout, + " - Searching for keys, allocating buffer for %llu keys\n", + (ulonglong) keys); + + if ((records= find_all_keys(info,keys,sort_keys,&buffpek,&maxbuffer, + &tempfile,&tempfile_for_exceptions)) + == HA_POS_ERROR) + goto err; /* purecov: tested */ + if (maxbuffer >= keys) + { + /* + merge_many_buff will crash if maxbuffer >= keys as then we cannot store in memory + the keys for each buffer. + */ + keys= maxbuffer + 1; + if (!(sort_keys= ((uchar **) + my_realloc(PSI_INSTRUMENT_ME, sort_keys, + (size_t) (keys*(sort_length+sizeof(char*))+ + HA_FT_MAXBYTELEN), MYF(MY_FREE_ON_ERROR))))) + goto err; + } + + if (maxbuffer == 0) + { + if (!no_messages) + my_fprintf(stdout, " - Dumping %llu keys\n", (ulonglong) records); + if (write_index(info, sort_keys, (ha_keys) records)) + goto err; /* purecov: inspected */ + } + else + { + keys=(keys*(sort_length+sizeof(char*)))/sort_length; + if (maxbuffer >= MERGEBUFF2) + { + if (!no_messages) + my_fprintf(stdout, " - Merging %llu keys\n", + (ulonglong) records); /* purecov: tested */ + if (merge_many_buff(info,keys,sort_keys, + dynamic_element(&buffpek,0,BUFFPEK *),&maxbuffer,&tempfile)) + goto err; /* purecov: inspected */ + } + if (flush_io_cache(&tempfile) || + reinit_io_cache(&tempfile,READ_CACHE,0L,0,0)) + goto err; /* purecov: inspected */ + if (!no_messages) + printf(" - Last merge and dumping keys\n"); /* purecov: tested */ + if (merge_index(info,keys,sort_keys,dynamic_element(&buffpek,0,BUFFPEK *), + maxbuffer,&tempfile)) + goto err; /* purecov: inspected */ + } + + if (flush_ft_buf(info) || flush_pending_blocks(info)) + goto err; + + if (my_b_inited(&tempfile_for_exceptions)) + { + MI_INFO *idx=info->sort_info->info; + uint keyno=info->key; + uint key_length, ref_length=idx->s->rec_reflength; + + if (!no_messages) + printf(" - Adding exceptions\n"); /* purecov: tested */ + if (flush_io_cache(&tempfile_for_exceptions) || + reinit_io_cache(&tempfile_for_exceptions,READ_CACHE,0L,0,0)) + goto err; + + while (!my_b_read(&tempfile_for_exceptions,(uchar*)&key_length, + sizeof(key_length)) + && !my_b_read(&tempfile_for_exceptions,(uchar*)sort_keys, + (uint) key_length)) + { + if (_mi_ck_write(idx,keyno,(uchar*) sort_keys,key_length-ref_length)) + goto err; + } + } + + error =0; + +err: + my_free(sort_keys); + delete_dynamic(&buffpek); + close_cached_file(&tempfile); + close_cached_file(&tempfile_for_exceptions); + + DBUG_RETURN(error ? -1 : 0); +} /* _create_index_by_sort */ + + +/* Search after all keys and place them in a temp. file */ + +static ha_rows find_all_keys(MI_SORT_PARAM *info, ha_rows keys, + uchar **sort_keys, DYNAMIC_ARRAY *buffpek, + size_t *maxbuffer, IO_CACHE *tempfile, + IO_CACHE *tempfile_for_exceptions) +{ + int error; + ha_rows idx; + DBUG_ENTER("find_all_keys"); + + idx=error=0; + sort_keys[0]=(uchar*) (sort_keys+keys); + + while (!(error=(*info->key_read)(info,sort_keys[idx]))) + { + if (info->real_key_length > info->key_length) + { + if (write_key(info,sort_keys[idx],tempfile_for_exceptions)) + DBUG_RETURN(HA_POS_ERROR); /* purecov: inspected */ + continue; + } + + if (++idx == keys) + { + if (info->write_keys(info,sort_keys,idx-1,(BUFFPEK *)alloc_dynamic(buffpek), + tempfile)) + DBUG_RETURN(HA_POS_ERROR); /* purecov: inspected */ + + sort_keys[0]=(uchar*) (sort_keys+keys); + memcpy(sort_keys[0],sort_keys[idx-1],(size_t) info->key_length); + idx=1; + } + sort_keys[idx]=sort_keys[idx-1]+info->key_length; + } + if (error > 0) + DBUG_RETURN(HA_POS_ERROR); /* Aborted by get_key */ /* purecov: inspected */ + if (buffpek->elements) + { + if (info->write_keys(info,sort_keys,idx,(BUFFPEK *)alloc_dynamic(buffpek), + tempfile)) + DBUG_RETURN(HA_POS_ERROR); /* purecov: inspected */ + *maxbuffer=buffpek->elements-1; + } + else + *maxbuffer=0; + + DBUG_RETURN((*maxbuffer)*(keys-1)+idx); +} /* find_all_keys */ + +static my_bool thr_find_all_keys_exec(MI_SORT_PARAM *sort_param) +{ + ulonglong memavl, old_memavl, sortbuff_size; + ha_keys UNINIT_VAR(keys), idx; + uint sort_length; + size_t maxbuffer; + uchar **sort_keys= NULL; + int error= 0; + DBUG_ENTER("thr_find_all_keys"); + DBUG_PRINT("enter", ("master: %d", sort_param->master)); + + if (sort_param->sort_info->got_error) + DBUG_RETURN(TRUE); + + set_sort_param_read_write(sort_param); + + my_b_clear(&sort_param->tempfile); + my_b_clear(&sort_param->tempfile_for_exceptions); + bzero((char*) &sort_param->buffpek, sizeof(sort_param->buffpek)); + bzero((char*) &sort_param->unique, sizeof(sort_param->unique)); + + sortbuff_size= sort_param->sortbuff_size; + memavl= MY_MAX(sortbuff_size, MIN_SORT_BUFFER); + idx= (ha_keys) sort_param->sort_info->max_records; + sort_length= sort_param->key_length; + maxbuffer= 1; + + while (memavl >= MIN_SORT_BUFFER) + { + if ((my_off_t) (idx+1)*(sort_length+sizeof(char*)) <= + (my_off_t) memavl) + keys= idx+1; + else if ((sort_param->sort_info->param->testflag & + (T_FORCE_SORT_MEMORY | T_CREATE_MISSING_KEYS)) == + T_FORCE_SORT_MEMORY) + { + /* + Use all of the given sort buffer for key data. + Allocate 1000 buffers at a start for new data. More buffers + will be allocated when needed. + */ + keys= memavl / (sort_length+sizeof(char*)); + maxbuffer= (size_t) MY_MIN((ulonglong) 1000, (idx / keys)+1); + } + else + { + size_t maxbuffer_org; + do + { + maxbuffer_org= maxbuffer; + if (memavl < sizeof(BUFFPEK)*maxbuffer || + (keys=(memavl-sizeof(BUFFPEK)*maxbuffer)/ + (sort_length+sizeof(char*))) <= 1 || + keys < maxbuffer) + { + mi_check_print_error(sort_param->sort_info->param, + "myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu rows: %llu sort_length: %u", + sortbuff_size, (ulonglong) idx, sort_length); + DBUG_RETURN(TRUE); + } + } + while ((maxbuffer= (size_t) (idx/(keys-1)+1)) != maxbuffer_org); + } + if ((sort_keys= (uchar**) my_malloc(PSI_INSTRUMENT_ME, + (size_t)(keys * (sort_length + sizeof(char*)) + + ((sort_param->keyinfo->flag & HA_FULLTEXT) ? + HA_FT_MAXBYTELEN : 0)), MYF(0)))) + { + if (my_init_dynamic_array(PSI_INSTRUMENT_ME, &sort_param->buffpek, + sizeof(BUFFPEK), maxbuffer, + MY_MIN(maxbuffer / 2, 1000), MYF(0))) + { + my_free(sort_keys); + sort_keys= NULL; /* Safety against double free on error. */ + } + else + break; + } + old_memavl= memavl; + if ((memavl= memavl / 4 * 3) < MIN_SORT_BUFFER && + old_memavl > MIN_SORT_BUFFER) + memavl= MIN_SORT_BUFFER; + } + if (memavl < MIN_SORT_BUFFER) + { + /* purecov: begin inspected */ + mi_check_print_error(sort_param->sort_info->param, + "myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu rows: %llu sort_length: %u", + sortbuff_size, (ulonglong) idx, sort_length); + my_errno= ENOMEM; + goto err; + /* purecov: end inspected */ + } + + if (sort_param->sort_info->param->testflag & T_VERBOSE) + my_fprintf(stdout, + "Key %d - Allocating buffer for %llu keys\n", + sort_param->key + 1, (ulonglong) keys); + sort_param->sort_keys= sort_keys; + + idx= error= 0; + sort_keys[0]= (uchar*) (sort_keys+keys); + + DBUG_PRINT("info", ("reading keys")); + while (!(error= sort_param->sort_info->got_error) && + !(error= (*sort_param->key_read)(sort_param, sort_keys[idx]))) + { + if (sort_param->real_key_length > sort_param->key_length) + { + if (write_key(sort_param, sort_keys[idx], + &sort_param->tempfile_for_exceptions)) + goto err; + continue; + } + + if (++idx == keys) + { + if (sort_param->write_keys(sort_param, sort_keys, idx - 1, + (BUFFPEK*) alloc_dynamic(&sort_param->buffpek), + &sort_param->tempfile)) + goto err; + sort_keys[0]= (uchar*) (sort_keys+keys); + memcpy(sort_keys[0], sort_keys[idx - 1], (size_t) sort_param->key_length); + idx= 1; + } + sort_keys[idx]= sort_keys[idx - 1] + sort_param->key_length; + } + + if (error > 0) + goto err; + + if (sort_param->buffpek.elements) + { + if (sort_param->write_keys(sort_param, sort_keys, idx, + (BUFFPEK*) alloc_dynamic(&sort_param->buffpek), + &sort_param->tempfile)) + goto err; + sort_param->keys= (sort_param->buffpek.elements - 1) * (keys - 1) + idx; + } + else + sort_param->keys= idx; + + DBUG_RETURN(FALSE); + +err: + DBUG_PRINT("error", ("got some error")); + sort_param->sort_info->got_error= 1; /* no need to protect with a mutex */ + my_free(sort_keys); + sort_param->sort_keys= 0; + delete_dynamic(& sort_param->buffpek); + close_cached_file(&sort_param->tempfile); + close_cached_file(&sort_param->tempfile_for_exceptions); + + DBUG_RETURN(TRUE); +} + +/* Search after all keys and place them in a temp. file */ + +pthread_handler_t thr_find_all_keys(void *arg) +{ + MI_SORT_PARAM *sort_param= (MI_SORT_PARAM*) arg; + my_bool error= FALSE; + /* If my_thread_init fails */ + if (my_thread_init() || thr_find_all_keys_exec(sort_param)) + error= TRUE; + + /* + Thread must clean up after itself. + */ + free_root(&sort_param->wordroot, MYF(0)); + /* + Detach from the share if the writer is involved. Avoid others to + be blocked. This includes a flush of the write buffer. This will + also indicate EOF to the readers. + That means that a writer always gets here first and readers - + only when they see EOF. But if a reader finishes prematurely + because of an error it may reach this earlier - don't allow it + to detach the writer thread. + */ + if (sort_param->master && sort_param->sort_info->info->rec_cache.share) + remove_io_thread(&sort_param->sort_info->info->rec_cache); + + /* Readers detach from the share if any. Avoid others to be blocked. */ + if (sort_param->read_cache.share) + remove_io_thread(&sort_param->read_cache); + + mysql_mutex_lock(&sort_param->sort_info->mutex); + if (error) + sort_param->sort_info->got_error= 1; + + if (!--sort_param->sort_info->threads_running) + mysql_cond_signal(&sort_param->sort_info->cond); + mysql_mutex_unlock(&sort_param->sort_info->mutex); + my_thread_end(); + return NULL; +} + + +int thr_write_keys(MI_SORT_PARAM *sort_param) +{ + MI_SORT_INFO *sort_info=sort_param->sort_info; + HA_CHECK *param=sort_info->param; + ulonglong UNINIT_VAR(length); + ha_rows keys; + ulong *rec_per_key_part=param->rec_per_key_part; + int got_error=sort_info->got_error; + uint i; + MI_INFO *info=sort_info->info; + MYISAM_SHARE *share=info->s; + MI_SORT_PARAM *sinfo; + uchar *mergebuf=0; + DBUG_ENTER("thr_write_keys"); + + for (i= 0, sinfo= sort_param ; + i < sort_info->total_keys ; + i++, sinfo++) + { + if (!sinfo->sort_keys) + { + got_error=1; + my_free(mi_get_rec_buff_ptr(info, sinfo->rec_buff)); + continue; + } + if (!got_error) + { + mi_set_key_active(share->state.key_map, sinfo->key); + if (!sinfo->buffpek.elements) + { + if (param->testflag & T_VERBOSE) + { + my_fprintf(stdout, + "Key %d - Dumping %llu keys\n", sinfo->key+1, + (ulonglong) sinfo->keys); + fflush(stdout); + } + if (write_index(sinfo, sinfo->sort_keys, sinfo->keys) || + flush_ft_buf(sinfo) || flush_pending_blocks(sinfo)) + got_error=1; + } + } + my_free(sinfo->sort_keys); + my_free(mi_get_rec_buff_ptr(info, sinfo->rec_buff)); + sinfo->sort_keys=0; + } + + for (i= 0, sinfo= sort_param ; + i < sort_info->total_keys ; + i++, + delete_dynamic(&sinfo->buffpek), + close_cached_file(&sinfo->tempfile), + close_cached_file(&sinfo->tempfile_for_exceptions), + rec_per_key_part+= sinfo->keyinfo->keysegs, sinfo++) + { + if (got_error) + continue; + + set_sort_param_read_write(sinfo); + + if (sinfo->buffpek.elements) + { + size_t maxbuffer=sinfo->buffpek.elements-1; + if (!mergebuf) + { + length=param->sort_buffer_length; + while (length >= MIN_SORT_BUFFER) + { + if ((mergebuf= my_malloc(PSI_INSTRUMENT_ME, + (size_t) length, MYF(0)))) + break; + length=length*3/4; + } + if (!mergebuf) + { + got_error=1; + continue; + } + } + keys=length/sinfo->key_length; + if (maxbuffer >= MERGEBUFF2) + { + if (param->testflag & T_VERBOSE) + my_fprintf(stdout, + "Key %d - Merging %llu keys\n", + sinfo->key+1, (ulonglong) sinfo->keys); + if (merge_many_buff(sinfo, keys, (uchar **)mergebuf, + dynamic_element(&sinfo->buffpek, 0, BUFFPEK *), + &maxbuffer, &sinfo->tempfile)) + { + got_error=1; + continue; + } + } + if (flush_io_cache(&sinfo->tempfile) || + reinit_io_cache(&sinfo->tempfile,READ_CACHE,0L,0,0)) + { + got_error=1; + continue; + } + if (param->testflag & T_VERBOSE) + printf("Key %d - Last merge and dumping keys\n", sinfo->key+1); + if (merge_index(sinfo, keys, (uchar **)mergebuf, + dynamic_element(&sinfo->buffpek,0,BUFFPEK *), + maxbuffer,&sinfo->tempfile) || + flush_ft_buf(sinfo) || + flush_pending_blocks(sinfo)) + { + got_error=1; + continue; + } + } + if (my_b_inited(&sinfo->tempfile_for_exceptions)) + { + uint key_length; + + if (param->testflag & T_VERBOSE) + printf("Key %d - Dumping 'long' keys\n", sinfo->key+1); + + if (flush_io_cache(&sinfo->tempfile_for_exceptions) || + reinit_io_cache(&sinfo->tempfile_for_exceptions,READ_CACHE,0L,0,0)) + { + got_error=1; + continue; + } + + while (!got_error && + !my_b_read(&sinfo->tempfile_for_exceptions,(uchar*)&key_length, + sizeof(key_length))) + { + uchar ft_buf[HA_FT_MAXBYTELEN + HA_FT_WLEN + 10]; + if (key_length > sizeof(ft_buf) || + my_b_read(&sinfo->tempfile_for_exceptions, (uchar*)ft_buf, + (uint)key_length) || + _mi_ck_write(info, sinfo->key, (uchar*)ft_buf, + key_length - info->s->rec_reflength)) + got_error=1; + } + } + if (!got_error && param->testflag & T_STATISTICS) + update_key_parts(sinfo->keyinfo, rec_per_key_part, sinfo->unique, + param->stats_method == MI_STATS_METHOD_IGNORE_NULLS ? + sinfo->notnull : NULL, + (ulonglong) info->state->records); + } + my_free(mergebuf); + DBUG_RETURN(got_error); +} + + /* Write all keys in memory to file for later merge */ + +static int write_keys(MI_SORT_PARAM *info, register uchar **sort_keys, + ha_keys count, BUFFPEK *buffpek, IO_CACHE *tempfile) +{ + uchar **end; + uint sort_length=info->key_length; + DBUG_ENTER("write_keys"); + + if (!buffpek) + DBUG_RETURN(1); /* Out of memory */ + + my_qsort2((uchar*) sort_keys,(size_t) count, sizeof(uchar*), + (qsort2_cmp) info->key_cmp, info); + if (!my_b_inited(tempfile) && + open_cached_file(tempfile, my_tmpdir(info->tmpdir), "ST", + DISK_BUFFER_SIZE, info->sort_info->param->myf_rw)) + DBUG_RETURN(1); /* purecov: inspected */ + + buffpek->file_pos=my_b_tell(tempfile); + buffpek->count=count; + + for (end=sort_keys+count ; sort_keys != end ; sort_keys++) + { + if (my_b_write(tempfile,(uchar*) *sort_keys, sort_length)) + DBUG_RETURN(1); /* purecov: inspected */ + } + DBUG_RETURN(0); +} /* write_keys */ + + +static inline int +my_var_write(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs) +{ + int err; + uint16 len = _mi_keylength(info->keyinfo, (uchar*) bufs); + + /* The following is safe as this is a local file */ + if ((err= my_b_write(to_file, (uchar*)&len, sizeof(len)))) + return (err); + if ((err= my_b_write(to_file,bufs, (uint) len))) + return (err); + return (0); +} + + +static int write_keys_varlen(MI_SORT_PARAM *info, + register uchar **sort_keys, + ha_keys count, BUFFPEK *buffpek, + IO_CACHE *tempfile) +{ + uchar **end; + int err; + DBUG_ENTER("write_keys_varlen"); + + if (!buffpek) + DBUG_RETURN(1); /* Out of memory */ + + my_qsort2((uchar*) sort_keys, (size_t) count, sizeof(uchar*), + (qsort2_cmp) info->key_cmp, info); + if (!my_b_inited(tempfile) && + open_cached_file(tempfile, my_tmpdir(info->tmpdir), "ST", + DISK_BUFFER_SIZE, info->sort_info->param->myf_rw)) + DBUG_RETURN(1); /* purecov: inspected */ + + buffpek->file_pos=my_b_tell(tempfile); + buffpek->count=count; + for (end=sort_keys+count ; sort_keys != end ; sort_keys++) + { + if ((err= my_var_write(info,tempfile, (uchar*) *sort_keys))) + DBUG_RETURN(err); + } + DBUG_RETURN(0); +} /* write_keys_varlen */ + + +static int write_key(MI_SORT_PARAM *info, uchar *key, IO_CACHE *tempfile) +{ + uint key_length=info->real_key_length; + DBUG_ENTER("write_key"); + + if (!my_b_inited(tempfile) && + open_cached_file(tempfile, my_tmpdir(info->tmpdir), "ST", + DISK_BUFFER_SIZE, info->sort_info->param->myf_rw)) + DBUG_RETURN(1); + + if (my_b_write(tempfile,(uchar*)&key_length,sizeof(key_length)) || + my_b_write(tempfile,(uchar*)key,(uint) key_length)) + DBUG_RETURN(1); + DBUG_RETURN(0); +} /* write_key */ + + +/* Write index */ + +static int write_index(MI_SORT_PARAM *info, register uchar **sort_keys, + register ha_keys count) +{ + DBUG_ENTER("write_index"); + + my_qsort2((uchar*) sort_keys,(size_t) count,sizeof(uchar*), + (qsort2_cmp) info->key_cmp,info); + while (count--) + { + if ((*info->key_write)(info,*sort_keys++)) + DBUG_RETURN(-1); /* purecov: inspected */ + } + DBUG_RETURN(0); +} /* write_index */ + + + /* Merge buffers to make < MERGEBUFF2 buffers */ + +static int merge_many_buff(MI_SORT_PARAM *info, ha_keys keys, + uchar **sort_keys, BUFFPEK *buffpek, + size_t *maxbuffer, IO_CACHE *t_file) +{ + register size_t i; + IO_CACHE t_file2, *from_file, *to_file, *temp; + BUFFPEK *lastbuff; + DBUG_ENTER("merge_many_buff"); + + if (*maxbuffer < MERGEBUFF2) + DBUG_RETURN(0); /* purecov: inspected */ + if (flush_io_cache(t_file) || + open_cached_file(&t_file2,my_tmpdir(info->tmpdir),"ST", + DISK_BUFFER_SIZE, info->sort_info->param->myf_rw)) + DBUG_RETURN(1); /* purecov: inspected */ + + from_file= t_file ; to_file= &t_file2; + while (*maxbuffer >= MERGEBUFF2) + { + reinit_io_cache(from_file,READ_CACHE,0L,0,0); + reinit_io_cache(to_file,WRITE_CACHE,0L,0,0); + lastbuff=buffpek; + for (i=0 ; i + MERGEBUFF*3/2 <= *maxbuffer ; i+=MERGEBUFF) + { + if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++, + buffpek+i,buffpek+i+MERGEBUFF-1)) + goto cleanup; + } + if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++, + buffpek+i,buffpek+ *maxbuffer)) + break; /* purecov: inspected */ + if (flush_io_cache(to_file)) + break; /* purecov: inspected */ + temp=from_file; from_file=to_file; to_file=temp; + *maxbuffer= (size_t) (lastbuff-buffpek)-1; + } +cleanup: + close_cached_file(to_file); /* This holds old result */ + if (to_file == t_file) + { + DBUG_ASSERT(t_file2.type == WRITE_CACHE); + *t_file=t_file2; /* Copy result file */ + } + + DBUG_RETURN(*maxbuffer >= MERGEBUFF2); /* Return 1 if interrupted */ +} /* merge_many_buff */ + + +/* + Read data to buffer + + SYNOPSIS + read_to_buffer() + fromfile File to read from + buffpek Where to read from + sort_length max length to read + RESULT + > 0 Number of bytes read + -1 Error +*/ + +static my_off_t read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek, + uint sort_length) +{ + register ha_keys count; + size_t length; + + if ((count= (ha_keys) MY_MIN((ha_rows) buffpek->max_keys, + (ha_rows) buffpek->count))) + { + if (my_b_pread(fromfile, (uchar*) buffpek->base, + (length= (size_t) (sort_length * count)), + buffpek->file_pos)) + return(HA_OFFSET_ERROR); + buffpek->key=buffpek->base; + buffpek->file_pos+= length; /* New filepos */ + buffpek->count-= count; + buffpek->mem_count= count; + } + return (((my_off_t) count) * sort_length); +} /* read_to_buffer */ + + +static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek, + uint sort_length) +{ + register ha_keys count; + uint16 length_of_key = 0; + uint idx; + uchar *buffp; + + if ((count= (ha_keys) MY_MIN((ha_rows) buffpek->max_keys,buffpek->count))) + { + buffp = buffpek->base; + + for (idx=1;idx<=count;idx++) + { + if (my_b_pread(fromfile, (uchar*)&length_of_key, + sizeof(length_of_key), buffpek->file_pos)) + return(HA_OFFSET_ERROR); + buffpek->file_pos+=sizeof(length_of_key); + if (my_b_pread(fromfile, (uchar*) buffp, + length_of_key, buffpek->file_pos)) + return(HA_OFFSET_ERROR); + buffpek->file_pos+=length_of_key; + buffp = buffp + sort_length; + } + buffpek->key=buffpek->base; + buffpek->count-= count; + buffpek->mem_count= count; + } + return (((my_off_t) count) * sort_length); +} /* read_to_buffer_varlen */ + + +static int write_merge_key_varlen(MI_SORT_PARAM *info, + IO_CACHE *to_file, uchar* key, + uint sort_length, ha_keys count) +{ + ha_keys idx; + uchar *bufs = key; + + for (idx=1;idx<=count;idx++) + { + int err; + if ((err= my_var_write(info, to_file, bufs))) + return (err); + bufs=bufs+sort_length; + } + return(0); +} + + +static int write_merge_key(MI_SORT_PARAM *info __attribute__((unused)), + IO_CACHE *to_file, uchar *key, + uint sort_length, ha_keys count) +{ + return my_b_write(to_file, key, (size_t) (sort_length * count)); +} + +/* + Merge buffers to one buffer + If to_file == 0 then use info->key_write + + Return: + 0 ok + 1 error +*/ + +static int +merge_buffers(MI_SORT_PARAM *info, ha_keys keys, IO_CACHE *from_file, + IO_CACHE *to_file, uchar **sort_keys, BUFFPEK *lastbuff, + BUFFPEK *Fb, BUFFPEK *Tb) +{ + int error= 1; + uint sort_length; + ha_keys maxcount; + ha_rows count; + my_off_t UNINIT_VAR(to_start_filepos), read_length; + uchar *strpos; + BUFFPEK *buffpek,**refpek; + QUEUE queue; + DBUG_ENTER("merge_buffers"); + + count=error=0; + maxcount= keys/((uint) (Tb-Fb) +1); + DBUG_ASSERT(maxcount > 0); + if (to_file) + to_start_filepos=my_b_tell(to_file); + strpos=(uchar*) sort_keys; + sort_length=info->key_length; + + if (init_queue(&queue,(uint) (Tb-Fb)+1,offsetof(BUFFPEK,key),0, + (int (*)(void*, uchar *,uchar*)) info->key_cmp, + (void*) info, 0, 0)) + DBUG_RETURN(1); /* purecov: inspected */ + + for (buffpek= Fb ; buffpek <= Tb ; buffpek++) + { + count+= buffpek->count; + buffpek->base= (uchar*) strpos; + buffpek->max_keys= maxcount; + strpos+= (read_length= info->read_to_buffer(from_file,buffpek, + sort_length)); + if (read_length == HA_OFFSET_ERROR) + goto err; /* purecov: inspected */ + queue_insert(&queue,(uchar*) buffpek); + } + + while (queue.elements > 1) + { + for (;;) + { + buffpek=(BUFFPEK*) queue_top(&queue); + if (to_file) + { + if (info->write_key(info,to_file,(uchar*) buffpek->key, + sort_length, 1)) + { + error=1; goto err; /* purecov: inspected */ + } + } + else + { + if ((*info->key_write)(info,(void*) buffpek->key)) + { + error=1; goto err; /* purecov: inspected */ + } + } + buffpek->key+=sort_length; + if (! --buffpek->mem_count) + { + /* It's enough to check for killedptr before a slow operation */ + if (killed_ptr(info->sort_info->param)) + { + goto err; + } + if (!(read_length= info->read_to_buffer(from_file,buffpek,sort_length))) + { + uchar *base= buffpek->base; + ha_keys max_keys=buffpek->max_keys; + + queue_remove_top(&queue); + + /* Put room used by buffer to use in other buffer */ + for (refpek= (BUFFPEK**) &queue_top(&queue); + refpek <= (BUFFPEK**) &queue_end(&queue); + refpek++) + { + buffpek= *refpek; + if (buffpek->base+buffpek->max_keys*sort_length == base) + { + buffpek->max_keys+=max_keys; + break; + } + else if (base+max_keys*sort_length == buffpek->base) + { + buffpek->base=base; + buffpek->max_keys+=max_keys; + break; + } + } + break; /* One buffer have been removed */ + } + else if (read_length == HA_OFFSET_ERROR) + goto err; /* purecov: inspected */ + } + queue_replace_top(&queue); /* Top element has been replaced */ + } + } + buffpek=(BUFFPEK*) queue_top(&queue); + buffpek->base= (uchar*) sort_keys; + buffpek->max_keys=keys; + do + { + if (to_file) + { + if (info->write_key(info,to_file,(uchar*) buffpek->key, + sort_length,buffpek->mem_count)) + { + error=1; goto err; /* purecov: inspected */ + } + } + else + { + register uchar *end; + strpos= (uchar*) buffpek->key; + for (end=strpos+buffpek->mem_count*sort_length; + strpos != end ; + strpos+=sort_length) + { + if ((*info->key_write)(info,(void*) strpos)) + { + error=1; goto err; /* purecov: inspected */ + } + } + } + } + while ((read_length= info->read_to_buffer(from_file,buffpek,sort_length)) != HA_OFFSET_ERROR && read_length != 0); + if (read_length == 0) + error= 0; + + lastbuff->count=count; + if (to_file) + lastbuff->file_pos=to_start_filepos; +err: + delete_queue(&queue); + DBUG_RETURN(error); +} /* merge_buffers */ + + + /* Do a merge to output-file (save only positions) */ + +static int +merge_index(MI_SORT_PARAM *info, ha_keys keys, uchar **sort_keys, + BUFFPEK *buffpek, size_t maxbuffer, IO_CACHE *tempfile) +{ + DBUG_ENTER("merge_index"); + if (merge_buffers(info,keys,tempfile,(IO_CACHE*) 0,sort_keys,buffpek,buffpek, + buffpek+maxbuffer)) + DBUG_RETURN(1); /* purecov: inspected */ + DBUG_RETURN(0); +} /* merge_index */ + + +static int +flush_ft_buf(MI_SORT_PARAM *info) +{ + int err=0; + if (info->sort_info->ft_buf) + { + err=sort_ft_buf_flush(info); + my_free(info->sort_info->ft_buf); + info->sort_info->ft_buf=0; + } + return err; +} diff --git a/storage/myisam/sp_defs.h b/storage/myisam/sp_defs.h new file mode 100644 index 00000000..d43fa49e --- /dev/null +++ b/storage/myisam/sp_defs.h @@ -0,0 +1,47 @@ +/* Copyright (c) 2002, 2004-2007 MySQL AB + Use is subject to license terms + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +#ifndef _SP_DEFS_H +#define _SP_DEFS_H + +#define SPDIMS 2 +#define SPTYPE HA_KEYTYPE_DOUBLE +#define SPLEN 8 + +#ifdef HAVE_SPATIAL + +enum wkbType +{ + wkbPoint = 1, + wkbLineString = 2, + wkbPolygon = 3, + wkbMultiPoint = 4, + wkbMultiLineString = 5, + wkbMultiPolygon = 6, + wkbGeometryCollection = 7 +}; + +enum wkbByteOrder +{ + wkbXDR = 0, /* Big Endian */ + wkbNDR = 1 /* Little Endian */ +}; + +uint sp_make_key(register MI_INFO *info, uint keynr, uchar *key, + const uchar *record, my_off_t filepos); + +#endif /*HAVE_SPATIAL*/ +#endif /* _SP_DEFS_H */ diff --git a/storage/myisam/sp_key.c b/storage/myisam/sp_key.c new file mode 100644 index 00000000..4c6ef759 --- /dev/null +++ b/storage/myisam/sp_key.c @@ -0,0 +1,284 @@ +/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "myisamdef.h" + +#ifdef HAVE_SPATIAL + +#include "sp_defs.h" + +static int sp_add_point_to_mbr(uchar *(*wkb), uchar *end, uint n_dims, + uchar byte_order, double *mbr); +static int sp_get_point_mbr(uchar *(*wkb), uchar *end, uint n_dims, + uchar byte_order, double *mbr); +static int sp_get_linestring_mbr(uchar *(*wkb), uchar *end, uint n_dims, + uchar byte_order, double *mbr); +static int sp_get_polygon_mbr(uchar *(*wkb), uchar *end, uint n_dims, + uchar byte_order, double *mbr); +static int sp_get_geometry_mbr(uchar *(*wkb), uchar *end, uint n_dims, + double *mbr, int top); +static int sp_mbr_from_wkb(uchar (*wkb), uint size, uint n_dims, double *mbr); + +uint sp_make_key(register MI_INFO *info, uint keynr, uchar *key, + const uchar *record, my_off_t filepos) +{ + HA_KEYSEG *keyseg; + MI_KEYDEF *keyinfo = &info->s->keyinfo[keynr]; + uint len = 0; + uchar *pos; + uint dlen; + uchar *dptr; + double mbr[SPDIMS * 2]; + uint i; + + keyseg = &keyinfo->seg[-1]; + pos = (uchar*)record + keyseg->start; + + dlen = _mi_calc_blob_length(keyseg->bit_start, pos); + memcpy(&dptr, pos + keyseg->bit_start, sizeof(char*)); + if (!dptr) + { + my_errno= HA_ERR_NULL_IN_SPATIAL; + return 0; + } + sp_mbr_from_wkb(dptr + 4, dlen - 4, SPDIMS, mbr); /* SRID */ + + for (i = 0, keyseg = keyinfo->seg; keyseg->type; keyseg++, i++) + { + uint length = keyseg->length, start= keyseg->start; + double val; + + DBUG_ASSERT(length == sizeof(double)); + DBUG_ASSERT(!(start % sizeof(double))); + DBUG_ASSERT(start < sizeof(mbr)); + DBUG_ASSERT(keyseg->type == HA_KEYTYPE_DOUBLE); + + val= mbr[start / sizeof (double)]; + if (isnan(val)) + { + bzero(key, length); + key+= length; + len+= length; + continue; + } + + if (keyseg->flag & HA_SWAP_KEY) + { + uchar buf[sizeof(double)]; + + float8store(buf, val); + pos= &buf[length]; + while (pos > buf) + *key++ = *--pos; + } + else + { + float8store((uchar *)key, val); + key += length; + } + len+= length; + } + _mi_dpointer(info, key, filepos); + return len; +} + +/* +Calculate minimal bounding rectangle (mbr) of the spatial object +stored in "well-known binary representation" (wkb) format. +*/ +static int sp_mbr_from_wkb(uchar *wkb, uint size, uint n_dims, double *mbr) +{ + uint i; + + for (i=0; i < n_dims; ++i) + { + mbr[i * 2] = DBL_MAX; + mbr[i * 2 + 1] = -DBL_MAX; + } + + return sp_get_geometry_mbr(&wkb, wkb + size, n_dims, mbr, 1); +} + +/* + Add one point stored in wkb to mbr +*/ + +static int sp_add_point_to_mbr(uchar *(*wkb), uchar *end, uint n_dims, + uchar byte_order __attribute__((unused)), + double *mbr) +{ + double ord; + double *mbr_end= mbr + n_dims * 2; + + while (mbr < mbr_end) + { + if ((*wkb) > end - 8) + return -1; + float8get(ord, (const uchar*) *wkb); + (*wkb)+= 8; + if (ord < *mbr) + *mbr= ord; + mbr++; + if (ord > *mbr) + *mbr= ord; + mbr++; + } + return 0; +} + + +static int sp_get_point_mbr(uchar *(*wkb), uchar *end, uint n_dims, + uchar byte_order, double *mbr) +{ + return sp_add_point_to_mbr(wkb, end, n_dims, byte_order, mbr); +} + + +static int sp_get_linestring_mbr(uchar *(*wkb), uchar *end, uint n_dims, + uchar byte_order, double *mbr) +{ + uint n_points; + + n_points = uint4korr(*wkb); + (*wkb) += 4; + for (; n_points > 0; --n_points) + { + /* Add next point to mbr */ + if (sp_add_point_to_mbr(wkb, end, n_dims, byte_order, mbr)) + return -1; + } + return 0; +} + + +static int sp_get_polygon_mbr(uchar *(*wkb), uchar *end, uint n_dims, + uchar byte_order, double *mbr) +{ + uint n_linear_rings; + uint n_points; + + n_linear_rings = uint4korr((*wkb)); + (*wkb) += 4; + + for (; n_linear_rings > 0; --n_linear_rings) + { + n_points = uint4korr((*wkb)); + (*wkb) += 4; + for (; n_points > 0; --n_points) + { + /* Add next point to mbr */ + if (sp_add_point_to_mbr(wkb, end, n_dims, byte_order, mbr)) + return -1; + } + } + return 0; +} + +static int sp_get_geometry_mbr(uchar *(*wkb), uchar *end, uint n_dims, + double *mbr, int top) +{ + int res; + uchar byte_order; + uint wkb_type; + + byte_order = *(*wkb); + ++(*wkb); + + wkb_type = uint4korr((*wkb)); + (*wkb) += 4; + + switch ((enum wkbType) wkb_type) + { + case wkbPoint: + res = sp_get_point_mbr(wkb, end, n_dims, byte_order, mbr); + break; + case wkbLineString: + res = sp_get_linestring_mbr(wkb, end, n_dims, byte_order, mbr); + break; + case wkbPolygon: + res = sp_get_polygon_mbr(wkb, end, n_dims, byte_order, mbr); + break; + case wkbMultiPoint: + { + uint n_items; + n_items = uint4korr((*wkb)); + (*wkb) += 4; + for (; n_items > 0; --n_items) + { + byte_order = *(*wkb); + ++(*wkb); + (*wkb) += 4; + if (sp_get_point_mbr(wkb, end, n_dims, byte_order, mbr)) + return -1; + } + res = 0; + break; + } + case wkbMultiLineString: + { + uint n_items; + n_items = uint4korr((*wkb)); + (*wkb) += 4; + for (; n_items > 0; --n_items) + { + byte_order = *(*wkb); + ++(*wkb); + (*wkb) += 4; + if (sp_get_linestring_mbr(wkb, end, n_dims, byte_order, mbr)) + return -1; + } + res = 0; + break; + } + case wkbMultiPolygon: + { + uint n_items; + n_items = uint4korr((*wkb)); + (*wkb) += 4; + for (; n_items > 0; --n_items) + { + byte_order = *(*wkb); + ++(*wkb); + (*wkb) += 4; + if (sp_get_polygon_mbr(wkb, end, n_dims, byte_order, mbr)) + return -1; + } + res = 0; + break; + } + case wkbGeometryCollection: + { + uint n_items; + + if (!top) + return -1; + + n_items = uint4korr((*wkb)); + (*wkb) += 4; + for (; n_items > 0; --n_items) + { + if (sp_get_geometry_mbr(wkb, end, n_dims, mbr, 0)) + return -1; + } + res = 0; + break; + } + default: + res = -1; + } + return res; +} + +#endif /*HAVE_SPATIAL*/ diff --git a/storage/myisam/sp_test.c b/storage/myisam/sp_test.c new file mode 100644 index 00000000..a33a6c8e --- /dev/null +++ b/storage/myisam/sp_test.c @@ -0,0 +1,498 @@ +/* Copyright (c) 2002, 2011, Oracle and/or its affiliates + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* Testing of the basic functions of a MyISAM spatial table */ +/* Written by Alex Barkov, who has a shared copyright to this code */ + +#include <my_global.h> +#include "myisam.h" + +#ifdef HAVE_SPATIAL +#include "sp_defs.h" + +#define MAX_REC_LENGTH 1024 +#define KEYALG HA_KEY_ALG_RTREE + +static void create_linestring(uchar *record,uint rownr); +static void print_record(uchar * record,my_off_t offs,const char * tail); + +static void create_key(uchar *key,uint rownr); +static void print_key(const uchar *key,const char * tail); + +static int run_test(const char *filename); +static int read_with_pos(MI_INFO * file, int silent); + +static int rtree_CreateLineStringWKB(double *ords, uint n_dims, uint n_points, + uchar *wkb); +static void rtree_PrintWKB(uchar *wkb, uint n_dims); + +static char blob_key[MAX_REC_LENGTH]; + + +int main(int argc __attribute__((unused)),char *argv[]) +{ + MY_INIT(argv[0]); + exit(run_test("sp_test")); +} + + +int run_test(const char *filename) +{ + MI_INFO *file; + MI_UNIQUEDEF uniquedef; + MI_CREATE_INFO create_info; + MI_COLUMNDEF recinfo[20]; + MI_KEYDEF keyinfo[20]; + HA_KEYSEG keyseg[20]; + key_range min_range, max_range; + int silent=0; + int create_flag=0; + int null_fields=0; + int nrecords=30; + int uniques=0; + int i; + int error; + int row_count=0; + uchar record[MAX_REC_LENGTH]; + uchar key[MAX_REC_LENGTH]; + uchar read_record[MAX_REC_LENGTH]; + int upd=10; + ha_rows hrows; + page_range pages; + + /* Define a column for NULLs and DEL markers*/ + + recinfo[0].type=FIELD_NORMAL; + recinfo[0].length=1; /* For NULL bits */ + + + /* Define spatial column */ + + recinfo[1].type=FIELD_BLOB; + recinfo[1].length=4 + portable_sizeof_char_ptr; + + + + /* Define a key with 1 spatial segment */ + + keyinfo[0].seg=keyseg; + keyinfo[0].keysegs=1; + keyinfo[0].flag=HA_SPATIAL; + keyinfo[0].key_alg=KEYALG; + + keyinfo[0].seg[0].type= HA_KEYTYPE_BINARY; + keyinfo[0].seg[0].flag=0; + keyinfo[0].seg[0].start= 1; + keyinfo[0].seg[0].length=1; /* Spatial ignores it anyway */ + keyinfo[0].seg[0].null_bit= null_fields ? 2 : 0; + keyinfo[0].seg[0].null_pos=0; + keyinfo[0].seg[0].language=default_charset_info->number; + keyinfo[0].seg[0].bit_start=4; /* Long BLOB */ + + + if (!silent) + printf("- Creating isam-file\n"); + + bzero((char*) &create_info,sizeof(create_info)); + create_info.max_rows=10000000; + + if (mi_create(filename, + 1, /* keys */ + keyinfo, + 2, /* columns */ + recinfo,uniques,&uniquedef,&create_info,create_flag)) + goto err; + + if (!silent) + printf("- Open isam-file\n"); + + if (!(file=mi_open(filename,2,HA_OPEN_ABORT_IF_LOCKED))) + goto err; + + if (!silent) + printf("- Writing key:s\n"); + + for (i=0; i<nrecords; i++ ) + { + create_linestring(record,i); + error=mi_write(file,record); + print_record(record,mi_position(file),"\n"); + if (!error) + { + row_count++; + } + else + { + printf("mi_write: %d\n", error); + goto err; + } + } + + if ((error=read_with_pos(file,silent))) + goto err; + + if (!silent) + printf("- Deleting rows with position\n"); + for (i=0; i < nrecords/4; i++) + { + my_errno=0; + bzero((char*) read_record,MAX_REC_LENGTH); + error=mi_rrnd(file,read_record,i == 0 ? 0L : HA_OFFSET_ERROR); + if (error) + { + printf("pos: %2d mi_rrnd: %3d errno: %3d\n",i,error,my_errno); + goto err; + } + print_record(read_record,mi_position(file),"\n"); + error=mi_delete(file,read_record); + if (error) + { + printf("pos: %2d mi_delete: %3d errno: %3d\n",i,error,my_errno); + goto err; + } + } + + if (!silent) + printf("- Updating rows with position\n"); + for (i=0; i < nrecords/2 ; i++) + { + my_errno=0; + bzero((char*) read_record,MAX_REC_LENGTH); + error=mi_rrnd(file,read_record,i == 0 ? 0L : HA_OFFSET_ERROR); + if (error) + { + if (error==HA_ERR_RECORD_DELETED) + continue; + printf("pos: %2d mi_rrnd: %3d errno: %3d\n",i,error,my_errno); + goto err; + } + print_record(read_record,mi_position(file),""); + create_linestring(record,i+nrecords*upd); + printf("\t-> "); + print_record(record,mi_position(file),"\n"); + error=mi_update(file,read_record,record); + if (error) + { + printf("pos: %2d mi_update: %3d errno: %3d\n",i,error,my_errno); + goto err; + } + } + + if ((error=read_with_pos(file,silent))) + goto err; + + if (!silent) + printf("- Test mi_rkey then a sequence of mi_rnext_same\n"); + + create_key(key, nrecords*4/5); + print_key(key," search for INTERSECT\n"); + + if ((error=mi_rkey(file,read_record,0,key,0,HA_READ_MBR_INTERSECT))) + { + printf("mi_rkey: %3d errno: %3d\n",error,my_errno); + goto err; + } + print_record(read_record,mi_position(file)," mi_rkey\n"); + row_count=1; + + for (;;) + { + if ((error=mi_rnext_same(file,read_record))) + { + if (error==HA_ERR_END_OF_FILE) + break; + printf("mi_next: %3d errno: %3d\n",error,my_errno); + goto err; + } + print_record(read_record,mi_position(file)," mi_rnext_same\n"); + row_count++; + } + printf(" %d rows\n",row_count); + + if (!silent) + printf("- Test mi_rfirst then a sequence of mi_rnext\n"); + + error=mi_rfirst(file,read_record,0); + if (error) + { + printf("mi_rfirst: %3d errno: %3d\n",error,my_errno); + goto err; + } + row_count=1; + print_record(read_record,mi_position(file)," mi_frirst\n"); + + for(i=0;i<nrecords;i++) { + if ((error=mi_rnext(file,read_record,0))) + { + if (error==HA_ERR_END_OF_FILE) + break; + printf("mi_next: %3d errno: %3d\n",error,my_errno); + goto err; + } + print_record(read_record,mi_position(file)," mi_rnext\n"); + row_count++; + } + printf(" %d rows\n",row_count); + + if (!silent) + printf("- Test mi_records_in_range()\n"); + + create_key(key, nrecords*upd); + print_key(key," INTERSECT\n"); + min_range.key= key; + min_range.length= 1000; /* Big enough */ + min_range.flag= HA_READ_MBR_INTERSECT; + max_range.key= record+1; + max_range.length= 1000; /* Big enough */ + max_range.flag= HA_READ_KEY_EXACT; + hrows= mi_records_in_range(file, 0, &min_range, &max_range, &pages); + printf(" %ld rows\n", (long) hrows); + + if (mi_close(file)) goto err; + my_end(MY_CHECK_ERROR); + return 0; + +err: + printf("got error: %3d when using myisam-database\n",my_errno); + return 1; /* skip warning */ +} + + +static int read_with_pos (MI_INFO * file,int silent) +{ + int error; + int i; + uchar read_record[MAX_REC_LENGTH]; + int rows=0; + + if (!silent) + printf("- Reading rows with position\n"); + for (i=0;;i++) + { + my_errno=0; + bzero((char*) read_record,MAX_REC_LENGTH); + error=mi_rrnd(file,read_record,i == 0 ? 0L : HA_OFFSET_ERROR); + if (error) + { + if (error==HA_ERR_END_OF_FILE) + break; + if (error==HA_ERR_RECORD_DELETED) + continue; + printf("pos: %2d mi_rrnd: %3d errno: %3d\n",i,error,my_errno); + return error; + } + rows++; + print_record(read_record,mi_position(file),"\n"); + } + printf(" %d rows\n",rows); + return 0; +} + + +static void print_record(uchar * record, my_off_t offs,const char * tail) +{ + uchar *pos; + char *ptr; + uint len; + + printf(" rec=(%d)",(unsigned char)record[0]); + pos=record+1; + len=sint4korr(pos); + pos+=4; + printf(" len=%d ",len); + memcpy(&ptr, pos, sizeof(char*)); + if (ptr) + rtree_PrintWKB((uchar*) ptr,SPDIMS); + else + printf("<NULL> "); + printf(" offs=%ld ",(long int)offs); + printf("%s",tail); +} + + +static void create_linestring(uchar *record,uint rownr) +{ + uint tmp; + char *ptr; + uchar *pos= record; + double x[200]; + int i,j; + int npoints=2; + + for(j=0;j<npoints;j++) + for(i=0;i<SPDIMS;i++) + x[i+j*SPDIMS]=rownr*j; + + bzero((char*) record,MAX_REC_LENGTH); + *pos=0x01; /* DEL marker */ + pos++; + + memset(blob_key,0,sizeof(blob_key)); + tmp=rtree_CreateLineStringWKB(x,SPDIMS,npoints, (uchar*) blob_key); + + int4store(pos,tmp); + pos+=4; + + ptr=blob_key; + memcpy(pos, &ptr, sizeof(char*)); +} + + +static void create_key(uchar *key,uint rownr) +{ + double c=rownr; + uchar *pos; + uint i; + + bzero(key,MAX_REC_LENGTH); + for (pos=key, i=0; i<2*SPDIMS; i++) + { + float8store(pos,c); + pos+=sizeof(c); + } +} + +static void print_key(const uchar *key,const char * tail) +{ + double c; + uint i; + + printf(" key="); + for (i=0; i<2*SPDIMS; i++) + { + float8get(c,key); + key+=sizeof(c); + printf("%.14g ",c); + } + printf("%s",tail); +} + + +static int rtree_CreateLineStringWKB(double *ords, uint n_dims, uint n_points, + uchar *wkb) +{ + uint i; + uint n_ords = n_dims * n_points; + + *wkb = wkbXDR; + ++wkb; + int4store(wkb, wkbLineString); + wkb += 4; + int4store(wkb, n_points); + wkb += 4; + for (i=0; i < n_ords; ++i) + { + float8store(wkb, ords[i]); + wkb += 8; + } + return 9 + n_points * n_dims * 8; +} + + +static void rtree_PrintWKB(uchar *wkb, uint n_dims) +{ + uint wkb_type; + + ++wkb; + wkb_type = uint4korr(wkb); + wkb += 4; + + switch ((enum wkbType)wkb_type) + { + case wkbPoint: + { + uint i; + double ord; + + printf("POINT("); + for (i=0; i < n_dims; ++i) + { + float8get(ord, wkb); + wkb += 8; + printf("%.14g", ord); + if (i < n_dims - 1) + printf(" "); + else + printf(")"); + } + break; + } + case wkbLineString: + { + uint p, i; + uint n_points; + double ord; + + printf("LineString("); + n_points = uint4korr(wkb); + wkb += 4; + for (p=0; p < n_points; ++p) + { + for (i=0; i < n_dims; ++i) + { + float8get(ord, wkb); + wkb += 8; + printf("%.14g", ord); + if (i < n_dims - 1) + printf(" "); + } + if (p < n_points - 1) + printf(", "); + else + printf(")"); + } + break; + } + case wkbPolygon: + { + printf("POLYGON(...)"); + break; + } + case wkbMultiPoint: + { + printf("MULTIPOINT(...)"); + break; + } + case wkbMultiLineString: + { + printf("MULTILINESTRING(...)"); + break; + } + case wkbMultiPolygon: + { + printf("MULTIPOLYGON(...)"); + break; + } + case wkbGeometryCollection: + { + printf("GEOMETRYCOLLECTION(...)"); + break; + } + default: + { + printf("UNKNOWN GEOMETRY TYPE"); + break; + } + } +} + +#else +int main(int argc __attribute__((unused)),char *argv[] __attribute__((unused))) +{ + exit(0); +} +#endif /*HAVE_SPATIAL*/ + +#include "mi_extrafunc.h" diff --git a/storage/myisam/test_pack b/storage/myisam/test_pack new file mode 100755 index 00000000..0cbeb57b --- /dev/null +++ b/storage/myisam/test_pack @@ -0,0 +1,9 @@ +silent="-s" +suffix=$MACH +mi_test1$MACH -s ; myisampack$MACH --force -s test1 ; myisamchk$MACH -es test1 ; myisamchk$MACH -rqs test1 ; myisamchk$MACH -es test1 ; myisamchk$MACH -us test1 ; myisamchk$MACH -es test1 +mi_test1$MACH -s -S ; myisampack$MACH --force -s test1 ; myisamchk$MACH -es test1 ; myisamchk$MACH -rqs test1 ; myisamchk$MACH -es test1 ;myisamchk$MACH -us test1 ; myisamchk$MACH -es test1 +mi_test1$MACH -s -b ; myisampack$MACH --force -s test1 ; myisamchk$MACH -es test1 ; myisamchk$MACH -rqs test1 ; myisamchk$MACH -es test1 +mi_test1$MACH -s -w ; myisampack$MACH --force -s test1 ; myisamchk$MACH -es test1 ; myisamchk$MACH -ros test1 ; myisamchk$MACH -es test1 + +mi_test2$MACH -s -t4 ; myisampack$MACH --force -s test2 ; myisamchk$MACH -es test2 ; myisamchk$MACH -ros test2 ; myisamchk$MACH -es test2 ; myisamchk$MACH -s -u test2 ; myisamchk$MACH -sm test2 +mi_test2$MACH -s -t4 -b -N ; myisampack$MACH --force -s test2 ; myisamchk$MACH -es test2 ; myisamchk$MACH -ros test2 ; myisamchk$MACH -es test2 ; myisamchk$MACH -s -u test2 ; myisamchk$MACH -sm test2 |