summaryrefslogtreecommitdiffstats
path: root/storage/maria/ma_ft_nlq_search.c
diff options
context:
space:
mode:
Diffstat (limited to 'storage/maria/ma_ft_nlq_search.c')
-rw-r--r--storage/maria/ma_ft_nlq_search.c388
1 files changed, 388 insertions, 0 deletions
diff --git a/storage/maria/ma_ft_nlq_search.c b/storage/maria/ma_ft_nlq_search.c
new file mode 100644
index 00000000..890de3db
--- /dev/null
+++ b/storage/maria/ma_ft_nlq_search.c
@@ -0,0 +1,388 @@
+/* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */
+
+/* Written by Sergei A. Golubchik, who has a shared copyright to this code */
+
+#define FT_CORE
+#include "ma_ftdefs.h"
+
+/* search with natural language queries */
+
+typedef struct ft_doc_rec
+{
+ my_off_t dpos;
+ double weight;
+} FT_DOC;
+
+struct st_ft_info
+{
+ struct _ft_vft *please;
+ MARIA_HA *info;
+ int ndocs;
+ int curdoc;
+ FT_DOC doc[1];
+};
+
+typedef struct st_all_in_one
+{
+ MARIA_HA *info;
+ uint keynr;
+ CHARSET_INFO *charset;
+ uchar *keybuff;
+ TREE dtree;
+} ALL_IN_ONE;
+
+typedef struct st_ft_superdoc
+{
+ FT_DOC doc;
+ FT_WORD *word_ptr;
+ double tmp_weight;
+} FT_SUPERDOC;
+
+
+static int FT_SUPERDOC_cmp(void* cmp_arg __attribute__((unused)),
+ FT_SUPERDOC *p1, FT_SUPERDOC *p2)
+{
+ if (p1->doc.dpos < p2->doc.dpos)
+ return -1;
+ if (p1->doc.dpos == p2->doc.dpos)
+ return 0;
+ return 1;
+}
+
+static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio)
+{
+ FT_WEIGTH subkeys;
+ int r;
+ uint doc_cnt;
+ FT_SUPERDOC sdoc, *sptr;
+ TREE_ELEMENT *selem;
+ double gweight=1;
+ MARIA_HA *info= aio->info;
+ MARIA_SHARE *share= info->s;
+ uchar *keybuff= aio->keybuff;
+ MARIA_KEYDEF *keyinfo= share->keyinfo+aio->keynr;
+ my_off_t key_root;
+ uint extra=HA_FT_WLEN+share->rec_reflength;
+ MARIA_KEY key;
+ float tmp_weight;
+ DBUG_ENTER("walk_and_match");
+
+ word->weight=LWS_FOR_QUERY;
+
+ _ma_ft_make_key(info, &key, aio->keynr, keybuff, word, 0);
+ key.data_length-= HA_FT_WLEN;
+ doc_cnt=0;
+ subkeys.i= 0;
+
+ if (share->lock_key_trees)
+ mysql_rwlock_rdlock(&share->keyinfo[aio->keynr].root_lock);
+
+ key_root= share->state.key_root[aio->keynr];
+
+ /* Skip rows inserted by current inserted */
+ for (r= _ma_search(info, &key, SEARCH_FIND, key_root) ;
+ !r &&
+ (subkeys.i= ft_sintXkorr(info->last_key.data +
+ info->last_key.data_length +
+ info->last_key.ref_length - extra)) > 0 &&
+ info->cur_row.lastpos >= info->state->data_file_length ;
+ r= _ma_search_next(info, &info->last_key, SEARCH_BIGGER, key_root))
+ ;
+
+ if (share->lock_key_trees)
+ mysql_rwlock_unlock(&share->keyinfo[aio->keynr].root_lock);
+
+ info->update|= HA_STATE_AKTIV; /* for _ma_test_if_changed() */
+
+ /* The following should be safe, even if we compare doubles */
+ while (!r && gweight)
+ {
+ if (key.data_length &&
+ ha_compare_word(aio->charset,
+ info->last_key.data + 1,
+ info->last_key.data_length +
+ info->last_key.ref_length - extra - 1,
+ key.data + 1, key.data_length - 1))
+ break;
+
+ if (subkeys.i < 0)
+ {
+ if (doc_cnt)
+ DBUG_RETURN(1); /* index is corrupted */
+ /*
+ TODO here: unsafe optimization, should this word
+ be skipped (based on subkeys) ?
+ */
+ keybuff+= key.data_length;
+ keyinfo= &share->ft2_keyinfo;
+ key_root= info->cur_row.lastpos;
+ key.data_length= 0;
+ if (share->lock_key_trees)
+ mysql_rwlock_rdlock(&share->keyinfo[aio->keynr].root_lock);
+ r= _ma_search_first(info, keyinfo, key_root);
+ goto do_skip;
+ }
+ /* The weight we read was actually a float */
+ tmp_weight= subkeys.f;
+ /* The following should be safe, even if we compare doubles */
+ if (tmp_weight==0)
+ DBUG_RETURN(doc_cnt); /* stopword, doc_cnt should be 0 */
+
+ sdoc.doc.dpos= info->cur_row.lastpos;
+
+ /* saving document matched into dtree */
+ if (!(selem=tree_insert(&aio->dtree, &sdoc, 0, aio->dtree.custom_arg)))
+ DBUG_RETURN(1);
+
+ sptr=(FT_SUPERDOC *)ELEMENT_KEY((&aio->dtree), selem);
+
+ if (selem->count==1) /* document's first match */
+ sptr->doc.weight=0;
+ else
+ sptr->doc.weight+=sptr->tmp_weight*sptr->word_ptr->weight;
+
+ sptr->word_ptr=word;
+ sptr->tmp_weight=tmp_weight;
+
+ doc_cnt++;
+
+ gweight=word->weight*GWS_IN_USE;
+ if (gweight < 0 || doc_cnt > 2000000)
+ gweight=0;
+
+ if (share->lock_key_trees)
+ mysql_rwlock_rdlock(&share->keyinfo[aio->keynr].root_lock);
+
+ if (_ma_test_if_changed(info) == 0)
+ r= _ma_search_next(info, &info->last_key, SEARCH_BIGGER, key_root);
+ else
+ r= _ma_search(info, &info->last_key, SEARCH_BIGGER, key_root);
+do_skip:
+ while ((subkeys.i= ft_sintXkorr(info->last_key.data +
+ info->last_key.data_length +
+ info->last_key.ref_length - extra)) > 0 &&
+ !r && info->cur_row.lastpos >= info->state->data_file_length)
+ r= _ma_search_next(info, &info->last_key, SEARCH_BIGGER, key_root);
+
+ if (share->lock_key_trees)
+ mysql_rwlock_unlock(&share->keyinfo[aio->keynr].root_lock);
+ }
+ word->weight=gweight;
+
+ DBUG_RETURN(0);
+}
+
+
+static int walk_and_copy(FT_SUPERDOC *from,
+ uint32 count __attribute__((unused)), FT_DOC **to)
+{
+ DBUG_ENTER("walk_and_copy");
+ from->doc.weight+=from->tmp_weight*from->word_ptr->weight;
+ (*to)->dpos=from->doc.dpos;
+ (*to)->weight=from->doc.weight;
+ (*to)++;
+ DBUG_RETURN(0);
+}
+
+static int walk_and_push(FT_SUPERDOC *from,
+ uint32 count __attribute__((unused)), QUEUE *best)
+{
+ DBUG_ENTER("walk_and_copy");
+ from->doc.weight+=from->tmp_weight*from->word_ptr->weight;
+ set_if_smaller(best->elements, ft_query_expansion_limit-1);
+ queue_insert(best, (uchar *)& from->doc);
+ DBUG_RETURN(0);
+}
+
+
+static int FT_DOC_cmp(void *unused __attribute__((unused)),
+ FT_DOC *a, FT_DOC *b)
+{
+ return CMP_NUM(b->weight, a->weight);
+}
+
+
+FT_INFO *maria_ft_init_nlq_search(MARIA_HA *info, uint keynr, uchar *query,
+ uint query_len, uint flags, uchar *record)
+{
+ TREE wtree;
+ ALL_IN_ONE aio;
+ FT_DOC *dptr;
+ FT_INFO *dlist=NULL;
+ MARIA_RECORD_POS saved_lastpos= info->cur_row.lastpos;
+ struct st_mysql_ftparser *parser;
+ MYSQL_FTPARSER_PARAM *ftparser_param;
+ DBUG_ENTER("maria_ft_init_nlq_search");
+
+ /* black magic ON */
+ if ((int) (keynr = _ma_check_index(info,keynr)) < 0)
+ DBUG_RETURN(NULL);
+ if (_ma_readinfo(info,F_RDLCK,1))
+ DBUG_RETURN(NULL);
+ /* black magic OFF */
+
+ aio.info=info;
+ aio.keynr=keynr;
+ aio.charset=info->s->keyinfo[keynr].seg->charset;
+ aio.keybuff= info->lastkey_buff2;
+ parser= info->s->keyinfo[keynr].parser;
+ if (! (ftparser_param= maria_ftparser_call_initializer(info, keynr, 0)))
+ goto err;
+
+ bzero(&wtree,sizeof(wtree));
+
+ init_tree(&aio.dtree,0,0,sizeof(FT_SUPERDOC),(qsort_cmp2)&FT_SUPERDOC_cmp,
+ NULL, NULL, MYF(0));
+
+ maria_ft_parse_init(&wtree, aio.charset);
+ ftparser_param->flags= 0;
+ if (maria_ft_parse(&wtree, query, query_len, parser, ftparser_param,
+ &wtree.mem_root))
+ goto err;
+
+ if (tree_walk(&wtree, (tree_walk_action)&walk_and_match, &aio,
+ left_root_right))
+ goto err;
+
+ if (flags & FT_EXPAND && ft_query_expansion_limit)
+ {
+ QUEUE best;
+ init_queue(&best,ft_query_expansion_limit,0,0, (queue_compare) &FT_DOC_cmp,
+ 0, 0, 0);
+ tree_walk(&aio.dtree, (tree_walk_action) &walk_and_push,
+ &best, left_root_right);
+ while (best.elements)
+ {
+ my_off_t docid= ((FT_DOC *)queue_remove_top(&best))->dpos;
+ if (!(*info->read_record)(info, record, docid))
+ {
+ info->update|= HA_STATE_AKTIV;
+ ftparser_param->flags= MYSQL_FTFLAGS_NEED_COPY;
+ if (unlikely(_ma_ft_parse(&wtree, info, keynr, record, ftparser_param,
+ &wtree.mem_root)))
+ {
+ delete_queue(&best);
+ goto err;
+ }
+ }
+ }
+ delete_queue(&best);
+ reset_tree(&aio.dtree);
+ if (tree_walk(&wtree, (tree_walk_action)&walk_and_match, &aio,
+ left_root_right))
+ goto err;
+
+ }
+
+ /*
+ If ndocs == 0, this will not allocate RAM for FT_INFO.doc[],
+ so if ndocs == 0, FT_INFO.doc[] must not be accessed.
+ */
+ dlist=(FT_INFO *)my_malloc(PSI_INSTRUMENT_ME, sizeof(FT_INFO)+
+ sizeof(FT_DOC)*
+ (int)(aio.dtree.elements_in_tree-1),
+ MYF(0));
+ if (!dlist)
+ goto err;
+
+ dlist->please= (struct _ft_vft *) & _ma_ft_vft_nlq;
+ dlist->ndocs=aio.dtree.elements_in_tree;
+ dlist->curdoc=-1;
+ dlist->info=aio.info;
+ dptr=dlist->doc;
+
+ tree_walk(&aio.dtree, (tree_walk_action) &walk_and_copy,
+ &dptr, left_root_right);
+
+ if (flags & FT_SORTED)
+ my_qsort2(dlist->doc, dlist->ndocs, sizeof(FT_DOC),
+ (qsort2_cmp)&FT_DOC_cmp, 0);
+
+err:
+ delete_tree(&aio.dtree, 0);
+ delete_tree(&wtree, 0);
+ info->cur_row.lastpos= saved_lastpos;
+ DBUG_RETURN(dlist);
+}
+
+
+int maria_ft_nlq_read_next(FT_INFO *handler, char *record)
+{
+ MARIA_HA *info= (MARIA_HA *) handler->info;
+
+ if (++handler->curdoc >= handler->ndocs)
+ {
+ --handler->curdoc;
+ return HA_ERR_END_OF_FILE;
+ }
+
+ info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
+
+ info->cur_row.lastpos= handler->doc[handler->curdoc].dpos;
+ if (!(*info->read_record)(info, (uchar *) record, info->cur_row.lastpos))
+ {
+ info->update|= HA_STATE_AKTIV; /* Record is read */
+ return 0;
+ }
+ return my_errno;
+}
+
+
+float maria_ft_nlq_find_relevance(FT_INFO *handler,
+ uchar *record __attribute__((unused)),
+ uint length __attribute__((unused)))
+{
+ int a,b,c;
+ FT_DOC *docs=handler->doc;
+ MARIA_RECORD_POS docid= handler->info->cur_row.lastpos;
+
+ if (docid == HA_POS_ERROR)
+ return -5.0;
+
+ /* Assuming docs[] is sorted by dpos... */
+
+ for (a=0, b=handler->ndocs, c=(a+b)/2; b-a>1; c=(a+b)/2)
+ {
+ if (docs[c].dpos > docid)
+ b=c;
+ else
+ a=c;
+ }
+ /* bounds check to avoid accessing unallocated handler->doc */
+ if (a < handler->ndocs && docs[a].dpos == docid)
+ return (float) docs[a].weight;
+ else
+ return 0.0;
+}
+
+
+void maria_ft_nlq_close_search(FT_INFO *handler)
+{
+ my_free(handler);
+}
+
+
+float maria_ft_nlq_get_relevance(FT_INFO *handler)
+{
+ return (float) handler->doc[handler->curdoc].weight;
+}
+
+
+void maria_ft_nlq_reinit_search(FT_INFO *handler)
+{
+ handler->curdoc=-1;
+}
+
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-21 16:34:06 +0000