diff options
Diffstat (limited to 'servers/slapd/back-bdb/tools.c')
| -rw-r--r-- | servers/slapd/back-bdb/tools.c | 1327 |
1 files changed, 1327 insertions, 0 deletions
diff --git a/servers/slapd/back-bdb/tools.c b/servers/slapd/back-bdb/tools.c new file mode 100644 index 0000000..9579d4c --- /dev/null +++ b/servers/slapd/back-bdb/tools.c @@ -0,0 +1,1327 @@ +/* tools.c - tools for slap tools */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software <http://www.openldap.org/>. + * + * Copyright 2000-2018 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +#include "portable.h" + +#include <stdio.h> +#include <ac/string.h> +#include <ac/errno.h> + +#define AVL_INTERNAL +#include "back-bdb.h" +#include "idl.h" + +static DBC *cursor = NULL; +static DBT key, data; +static EntryHeader eh; +static ID nid, previd = NOID; +static char ehbuf[16]; + +typedef struct dn_id { + ID id; + struct berval dn; +} dn_id; + +#define HOLE_SIZE 4096 +static dn_id hbuf[HOLE_SIZE], *holes = hbuf; +static unsigned nhmax = HOLE_SIZE; +static unsigned nholes; + +static int index_nattrs; + +static struct berval *tool_base; +static int tool_scope; +static Filter *tool_filter; +static Entry *tool_next_entry; + +#ifdef BDB_TOOL_IDL_CACHING +#define bdb_tool_idl_cmp BDB_SYMBOL(tool_idl_cmp) +#define bdb_tool_idl_flush_one BDB_SYMBOL(tool_idl_flush_one) +#define bdb_tool_idl_flush BDB_SYMBOL(tool_idl_flush) + +static int bdb_tool_idl_flush( BackendDB *be ); + +#define IDBLOCK 1024 + +typedef struct bdb_tool_idl_cache_entry { + struct bdb_tool_idl_cache_entry *next; + ID ids[IDBLOCK]; +} bdb_tool_idl_cache_entry; + +typedef struct bdb_tool_idl_cache { + struct berval kstr; + bdb_tool_idl_cache_entry *head, *tail; + ID first, last; + int count; +} bdb_tool_idl_cache; + +static bdb_tool_idl_cache_entry *bdb_tool_idl_free_list; +#endif /* BDB_TOOL_IDL_CACHING */ + +static ID bdb_tool_ix_id; +static Operation *bdb_tool_ix_op; +static int *bdb_tool_index_threads, bdb_tool_index_tcount; +static void *bdb_tool_index_rec; +static struct bdb_info *bdb_tool_info; +static ldap_pvt_thread_mutex_t bdb_tool_index_mutex; +static ldap_pvt_thread_cond_t bdb_tool_index_cond_main; +static ldap_pvt_thread_cond_t bdb_tool_index_cond_work; + +#if DB_VERSION_FULL >= 0x04060000 +#define USE_TRICKLE 1 +#else +/* Seems to slow things down too much in BDB 4.5 */ +#undef USE_TRICKLE +#endif + +#ifdef USE_TRICKLE +static ldap_pvt_thread_mutex_t bdb_tool_trickle_mutex; +static ldap_pvt_thread_cond_t bdb_tool_trickle_cond; +static ldap_pvt_thread_cond_t bdb_tool_trickle_cond_end; + +static void * bdb_tool_trickle_task( void *ctx, void *ptr ); +static int bdb_tool_trickle_active; +#endif + +static void * bdb_tool_index_task( void *ctx, void *ptr ); + +static int +bdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep ); + +static int bdb_tool_threads; + +int bdb_tool_entry_open( + BackendDB *be, int mode ) +{ + struct bdb_info *bdb = (struct bdb_info *) be->be_private; + + /* initialize key and data thangs */ + DBTzero( &key ); + DBTzero( &data ); + key.flags = DB_DBT_USERMEM; + key.data = &nid; + key.size = key.ulen = sizeof( nid ); + data.flags = DB_DBT_USERMEM; + + if (cursor == NULL) { + int rc = bdb->bi_id2entry->bdi_db->cursor( + bdb->bi_id2entry->bdi_db, bdb->bi_cache.c_txn, &cursor, + bdb->bi_db_opflags ); + if( rc != 0 ) { + return -1; + } + } + + /* Set up for threaded slapindex */ + if (( slapMode & (SLAP_TOOL_QUICK|SLAP_TOOL_READONLY)) == SLAP_TOOL_QUICK ) { + if ( !bdb_tool_info ) { +#ifdef USE_TRICKLE + ldap_pvt_thread_mutex_init( &bdb_tool_trickle_mutex ); + ldap_pvt_thread_cond_init( &bdb_tool_trickle_cond ); + ldap_pvt_thread_cond_init( &bdb_tool_trickle_cond_end ); + ldap_pvt_thread_pool_submit( &connection_pool, bdb_tool_trickle_task, bdb->bi_dbenv ); +#endif + + ldap_pvt_thread_mutex_init( &bdb_tool_index_mutex ); + ldap_pvt_thread_cond_init( &bdb_tool_index_cond_main ); + ldap_pvt_thread_cond_init( &bdb_tool_index_cond_work ); + if ( bdb->bi_nattrs ) { + int i; + bdb_tool_threads = slap_tool_thread_max - 1; + if ( bdb_tool_threads > 1 ) { + bdb_tool_index_threads = ch_malloc( bdb_tool_threads * sizeof( int )); + bdb_tool_index_rec = ch_malloc( bdb->bi_nattrs * sizeof( IndexRec )); + bdb_tool_index_tcount = bdb_tool_threads - 1; + for (i=1; i<bdb_tool_threads; i++) { + int *ptr = ch_malloc( sizeof( int )); + *ptr = i; + ldap_pvt_thread_pool_submit( &connection_pool, + bdb_tool_index_task, ptr ); + } + } + } + bdb_tool_info = bdb; + } + } + + return 0; +} + +int bdb_tool_entry_close( + BackendDB *be ) +{ + if ( bdb_tool_info ) { + slapd_shutdown = 1; +#ifdef USE_TRICKLE + ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex ); + + /* trickle thread may not have started yet */ + while ( !bdb_tool_trickle_active ) + ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond_end, + &bdb_tool_trickle_mutex ); + + ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond ); + while ( bdb_tool_trickle_active ) + ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond_end, + &bdb_tool_trickle_mutex ); + ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex ); +#endif + if ( bdb_tool_threads > 1 ) { + ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex ); + + /* There might still be some threads starting */ + while ( bdb_tool_index_tcount > 0 ) { + ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main, + &bdb_tool_index_mutex ); + } + + bdb_tool_index_tcount = bdb_tool_threads - 1; + ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work ); + + /* Make sure all threads are stopped */ + while ( bdb_tool_index_tcount > 0 ) { + ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main, + &bdb_tool_index_mutex ); + } + ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex ); + + ch_free( bdb_tool_index_threads ); + ch_free( bdb_tool_index_rec ); + bdb_tool_index_tcount = bdb_tool_threads - 1; + } + bdb_tool_info = NULL; + slapd_shutdown = 0; + } + + if( eh.bv.bv_val ) { + ch_free( eh.bv.bv_val ); + eh.bv.bv_val = NULL; + } + + if( cursor ) { + cursor->c_close( cursor ); + cursor = NULL; + } + +#ifdef BDB_TOOL_IDL_CACHING + bdb_tool_idl_flush( be ); +#endif + + if( nholes ) { + unsigned i; + fprintf( stderr, "Error, entries missing!\n"); + for (i=0; i<nholes; i++) { + fprintf(stderr, " entry %ld: %s\n", + holes[i].id, holes[i].dn.bv_val); + } + return -1; + } + + return 0; +} + +ID +bdb_tool_entry_first_x( + BackendDB *be, + struct berval *base, + int scope, + Filter *f ) +{ + tool_base = base; + tool_scope = scope; + tool_filter = f; + + return bdb_tool_entry_next( be ); +} + +ID bdb_tool_entry_next( + BackendDB *be ) +{ + int rc; + ID id; + struct bdb_info *bdb; + + assert( be != NULL ); + assert( slapMode & SLAP_TOOL_MODE ); + + bdb = (struct bdb_info *) be->be_private; + assert( bdb != NULL ); + +next:; + /* Get the header */ + data.ulen = data.dlen = sizeof( ehbuf ); + data.data = ehbuf; + data.flags |= DB_DBT_PARTIAL; + rc = cursor->c_get( cursor, &key, &data, DB_NEXT ); + + if( rc ) { + /* If we're doing linear indexing and there are more attrs to + * index, and we're at the end of the database, start over. + */ + if ( index_nattrs && rc == DB_NOTFOUND ) { + /* optional - do a checkpoint here? */ + bdb_attr_info_free( bdb->bi_attrs[0] ); + bdb->bi_attrs[0] = bdb->bi_attrs[index_nattrs]; + index_nattrs--; + rc = cursor->c_get( cursor, &key, &data, DB_FIRST ); + if ( rc ) { + return NOID; + } + } else { + return NOID; + } + } + + BDB_DISK2ID( key.data, &id ); + previd = id; + + if ( tool_filter || tool_base ) { + static Operation op = {0}; + static Opheader ohdr = {0}; + + op.o_hdr = &ohdr; + op.o_bd = be; + op.o_tmpmemctx = NULL; + op.o_tmpmfuncs = &ch_mfuncs; + + if ( tool_next_entry ) { + bdb_entry_release( &op, tool_next_entry, 0 ); + tool_next_entry = NULL; + } + + rc = bdb_tool_entry_get_int( be, id, &tool_next_entry ); + if ( rc == LDAP_NO_SUCH_OBJECT ) { + goto next; + } + + assert( tool_next_entry != NULL ); + +#ifdef BDB_HIER + /* TODO: needed until BDB_HIER is handled accordingly + * in bdb_tool_entry_get_int() */ + if ( tool_base && !dnIsSuffixScope( &tool_next_entry->e_nname, tool_base, tool_scope ) ) + { + bdb_entry_release( &op, tool_next_entry, 0 ); + tool_next_entry = NULL; + goto next; + } +#endif + + if ( tool_filter && test_filter( NULL, tool_next_entry, tool_filter ) != LDAP_COMPARE_TRUE ) + { + bdb_entry_release( &op, tool_next_entry, 0 ); + tool_next_entry = NULL; + goto next; + } + } + + return id; +} + +ID bdb_tool_dn2id_get( + Backend *be, + struct berval *dn +) +{ + Operation op = {0}; + Opheader ohdr = {0}; + EntryInfo *ei = NULL; + int rc; + + if ( BER_BVISEMPTY(dn) ) + return 0; + + op.o_hdr = &ohdr; + op.o_bd = be; + op.o_tmpmemctx = NULL; + op.o_tmpmfuncs = &ch_mfuncs; + + rc = bdb_cache_find_ndn( &op, 0, dn, &ei ); + if ( ei ) bdb_cache_entryinfo_unlock( ei ); + if ( rc == DB_NOTFOUND ) + return NOID; + + return ei->bei_id; +} + +static int +bdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep ) +{ + Entry *e = NULL; + char *dptr; + int rc, eoff; + + assert( be != NULL ); + assert( slapMode & SLAP_TOOL_MODE ); + + if ( ( tool_filter || tool_base ) && id == previd && tool_next_entry != NULL ) { + *ep = tool_next_entry; + tool_next_entry = NULL; + return LDAP_SUCCESS; + } + + if ( id != previd ) { + data.ulen = data.dlen = sizeof( ehbuf ); + data.data = ehbuf; + data.flags |= DB_DBT_PARTIAL; + + BDB_ID2DISK( id, &nid ); + rc = cursor->c_get( cursor, &key, &data, DB_SET ); + if ( rc ) { + rc = LDAP_OTHER; + goto done; + } + } + + /* Get the header */ + dptr = eh.bv.bv_val; + eh.bv.bv_val = ehbuf; + eh.bv.bv_len = data.size; + rc = entry_header( &eh ); + eoff = eh.data - eh.bv.bv_val; + eh.bv.bv_val = dptr; + if ( rc ) { + rc = LDAP_OTHER; + goto done; + } + + /* Get the size */ + data.flags &= ~DB_DBT_PARTIAL; + data.ulen = 0; + rc = cursor->c_get( cursor, &key, &data, DB_CURRENT ); + if ( rc != DB_BUFFER_SMALL ) { + rc = LDAP_OTHER; + goto done; + } + + /* Allocate a block and retrieve the data */ + eh.bv.bv_len = eh.nvals * sizeof( struct berval ) + data.size; + eh.bv.bv_val = ch_realloc( eh.bv.bv_val, eh.bv.bv_len ); + eh.data = eh.bv.bv_val + eh.nvals * sizeof( struct berval ); + data.data = eh.data; + data.ulen = data.size; + + /* Skip past already parsed nattr/nvals */ + eh.data += eoff; + + rc = cursor->c_get( cursor, &key, &data, DB_CURRENT ); + if ( rc ) { + rc = LDAP_OTHER; + goto done; + } + +#ifndef BDB_HIER + /* TODO: handle BDB_HIER accordingly */ + if ( tool_base != NULL ) { + struct berval ndn; + entry_decode_dn( &eh, NULL, &ndn ); + + if ( !dnIsSuffixScope( &ndn, tool_base, tool_scope ) ) { + return LDAP_NO_SUCH_OBJECT; + } + } +#endif + +#ifdef SLAP_ZONE_ALLOC + /* FIXME: will add ctx later */ + rc = entry_decode( &eh, &e, NULL ); +#else + rc = entry_decode( &eh, &e ); +#endif + + if( rc == LDAP_SUCCESS ) { + e->e_id = id; +#ifdef BDB_HIER + if ( slapMode & SLAP_TOOL_READONLY ) { + struct bdb_info *bdb = (struct bdb_info *) be->be_private; + EntryInfo *ei = NULL; + Operation op = {0}; + Opheader ohdr = {0}; + + op.o_hdr = &ohdr; + op.o_bd = be; + op.o_tmpmemctx = NULL; + op.o_tmpmfuncs = &ch_mfuncs; + + rc = bdb_cache_find_parent( &op, bdb->bi_cache.c_txn, id, &ei ); + if ( rc == LDAP_SUCCESS ) { + bdb_cache_entryinfo_unlock( ei ); + e->e_private = ei; + ei->bei_e = e; + bdb_fix_dn( e, 0 ); + ei->bei_e = NULL; + e->e_private = NULL; + } + } +#endif + } +done: + if ( e != NULL ) { + *ep = e; + } + + return rc; +} + +Entry* +bdb_tool_entry_get( BackendDB *be, ID id ) +{ + Entry *e = NULL; + + (void)bdb_tool_entry_get_int( be, id, &e ); + return e; +} + +static int bdb_tool_next_id( + Operation *op, + DB_TXN *tid, + Entry *e, + struct berval *text, + int hole ) +{ + struct berval dn = e->e_name; + struct berval ndn = e->e_nname; + struct berval pdn, npdn; + EntryInfo *ei = NULL, eidummy; + int rc; + + if (ndn.bv_len == 0) { + e->e_id = 0; + return 0; + } + + rc = bdb_cache_find_ndn( op, tid, &ndn, &ei ); + if ( ei ) bdb_cache_entryinfo_unlock( ei ); + if ( rc == DB_NOTFOUND ) { + if ( !be_issuffix( op->o_bd, &ndn ) ) { + ID eid = e->e_id; + dnParent( &dn, &pdn ); + dnParent( &ndn, &npdn ); + e->e_name = pdn; + e->e_nname = npdn; + rc = bdb_tool_next_id( op, tid, e, text, 1 ); + e->e_name = dn; + e->e_nname = ndn; + if ( rc ) { + return rc; + } + /* If parent didn't exist, it was created just now + * and its ID is now in e->e_id. Make sure the current + * entry gets added under the new parent ID. + */ + if ( eid != e->e_id ) { + eidummy.bei_id = e->e_id; + ei = &eidummy; + } + } + rc = bdb_next_id( op->o_bd, &e->e_id ); + if ( rc ) { + snprintf( text->bv_val, text->bv_len, + "next_id failed: %s (%d)", + db_strerror(rc), rc ); + Debug( LDAP_DEBUG_ANY, + "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 ); + return rc; + } + rc = bdb_dn2id_add( op, tid, ei, e ); + if ( rc ) { + snprintf( text->bv_val, text->bv_len, + "dn2id_add failed: %s (%d)", + db_strerror(rc), rc ); + Debug( LDAP_DEBUG_ANY, + "=> bdb_tool_next_id: %s\n", text->bv_val, 0, 0 ); + } else if ( hole ) { + if ( nholes == nhmax - 1 ) { + if ( holes == hbuf ) { + holes = ch_malloc( nhmax * sizeof(dn_id) * 2 ); + AC_MEMCPY( holes, hbuf, sizeof(hbuf) ); + } else { + holes = ch_realloc( holes, nhmax * sizeof(dn_id) * 2 ); + } + nhmax *= 2; + } + ber_dupbv( &holes[nholes].dn, &ndn ); + holes[nholes++].id = e->e_id; + } + } else if ( !hole ) { + unsigned i, j; + + e->e_id = ei->bei_id; + + for ( i=0; i<nholes; i++) { + if ( holes[i].id == e->e_id ) { + free(holes[i].dn.bv_val); + for (j=i;j<nholes;j++) holes[j] = holes[j+1]; + holes[j].id = 0; + nholes--; + break; + } else if ( holes[i].id > e->e_id ) { + break; + } + } + } + return rc; +} + +static int +bdb_tool_index_add( + Operation *op, + DB_TXN *txn, + Entry *e ) +{ + struct bdb_info *bdb = (struct bdb_info *) op->o_bd->be_private; + + if ( !bdb->bi_nattrs ) + return 0; + + if ( bdb_tool_threads > 1 ) { + IndexRec *ir; + int i, rc; + Attribute *a; + + ir = bdb_tool_index_rec; + memset(ir, 0, bdb->bi_nattrs * sizeof( IndexRec )); + + for ( a = e->e_attrs; a != NULL; a = a->a_next ) { + rc = bdb_index_recset( bdb, a, a->a_desc->ad_type, + &a->a_desc->ad_tags, ir ); + if ( rc ) + return rc; + } + bdb_tool_ix_id = e->e_id; + bdb_tool_ix_op = op; + ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex ); + /* Wait for all threads to be ready */ + while ( bdb_tool_index_tcount > 0 ) { + ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main, + &bdb_tool_index_mutex ); + } + for ( i=1; i<bdb_tool_threads; i++ ) + bdb_tool_index_threads[i] = LDAP_BUSY; + bdb_tool_index_tcount = bdb_tool_threads - 1; + ldap_pvt_thread_cond_broadcast( &bdb_tool_index_cond_work ); + ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex ); + rc = bdb_index_recrun( op, bdb, ir, e->e_id, 0 ); + if ( rc ) + return rc; + ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex ); + for ( i=1; i<bdb_tool_threads; i++ ) { + if ( bdb_tool_index_threads[i] == LDAP_BUSY ) { + ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_main, + &bdb_tool_index_mutex ); + i--; + continue; + } + if ( bdb_tool_index_threads[i] ) { + rc = bdb_tool_index_threads[i]; + break; + } + } + ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex ); + return rc; + } else { + return bdb_index_entry_add( op, txn, e ); + } +} + +ID bdb_tool_entry_put( + BackendDB *be, + Entry *e, + struct berval *text ) +{ + int rc; + struct bdb_info *bdb; + DB_TXN *tid = NULL; + Operation op = {0}; + Opheader ohdr = {0}; + + assert( be != NULL ); + assert( slapMode & SLAP_TOOL_MODE ); + + assert( text != NULL ); + assert( text->bv_val != NULL ); + assert( text->bv_val[0] == '\0' ); /* overconservative? */ + + Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(bdb_tool_entry_put) + "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 ); + + bdb = (struct bdb_info *) be->be_private; + + if (! (slapMode & SLAP_TOOL_QUICK)) { + rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid, + bdb->bi_db_opflags ); + if( rc != 0 ) { + snprintf( text->bv_val, text->bv_len, + "txn_begin failed: %s (%d)", + db_strerror(rc), rc ); + Debug( LDAP_DEBUG_ANY, + "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n", + text->bv_val, 0, 0 ); + return NOID; + } + Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(bdb_tool_entry_put) ": txn id: %x\n", + tid->id(tid), 0, 0 ); + } + + op.o_hdr = &ohdr; + op.o_bd = be; + op.o_tmpmemctx = NULL; + op.o_tmpmfuncs = &ch_mfuncs; + + /* add dn2id indices */ + rc = bdb_tool_next_id( &op, tid, e, text, 0 ); + if( rc != 0 ) { + goto done; + } + +#ifdef USE_TRICKLE + if (( slapMode & SLAP_TOOL_QUICK ) && (( e->e_id & 0xfff ) == 0xfff )) { + ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond ); + } +#endif + + if ( !bdb->bi_linear_index ) + rc = bdb_tool_index_add( &op, tid, e ); + if( rc != 0 ) { + snprintf( text->bv_val, text->bv_len, + "index_entry_add failed: %s (%d)", + rc == LDAP_OTHER ? "Internal error" : + db_strerror(rc), rc ); + Debug( LDAP_DEBUG_ANY, + "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n", + text->bv_val, 0, 0 ); + goto done; + } + + /* id2entry index */ + rc = bdb_id2entry_add( be, tid, e ); + if( rc != 0 ) { + snprintf( text->bv_val, text->bv_len, + "id2entry_add failed: %s (%d)", + db_strerror(rc), rc ); + Debug( LDAP_DEBUG_ANY, + "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n", + text->bv_val, 0, 0 ); + goto done; + } + +done: + if( rc == 0 ) { + if ( !( slapMode & SLAP_TOOL_QUICK )) { + rc = TXN_COMMIT( tid, 0 ); + if( rc != 0 ) { + snprintf( text->bv_val, text->bv_len, + "txn_commit failed: %s (%d)", + db_strerror(rc), rc ); + Debug( LDAP_DEBUG_ANY, + "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n", + text->bv_val, 0, 0 ); + e->e_id = NOID; + } + } + + } else { + if ( !( slapMode & SLAP_TOOL_QUICK )) { + TXN_ABORT( tid ); + snprintf( text->bv_val, text->bv_len, + "txn_aborted! %s (%d)", + rc == LDAP_OTHER ? "Internal error" : + db_strerror(rc), rc ); + Debug( LDAP_DEBUG_ANY, + "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n", + text->bv_val, 0, 0 ); + } + e->e_id = NOID; + } + + return e->e_id; +} + +int bdb_tool_entry_reindex( + BackendDB *be, + ID id, + AttributeDescription **adv ) +{ + struct bdb_info *bi = (struct bdb_info *) be->be_private; + int rc; + Entry *e; + DB_TXN *tid = NULL; + Operation op = {0}; + Opheader ohdr = {0}; + + Debug( LDAP_DEBUG_ARGS, + "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld )\n", + (long) id, 0, 0 ); + assert( tool_base == NULL ); + assert( tool_filter == NULL ); + + /* No indexes configured, nothing to do. Could return an + * error here to shortcut things. + */ + if (!bi->bi_attrs) { + return 0; + } + + /* Check for explicit list of attrs to index */ + if ( adv ) { + int i, j, n; + + if ( bi->bi_attrs[0]->ai_desc != adv[0] ) { + /* count */ + for ( n = 0; adv[n]; n++ ) ; + + /* insertion sort */ + for ( i = 0; i < n; i++ ) { + AttributeDescription *ad = adv[i]; + for ( j = i-1; j>=0; j--) { + if ( SLAP_PTRCMP( adv[j], ad ) <= 0 ) break; + adv[j+1] = adv[j]; + } + adv[j+1] = ad; + } + } + + for ( i = 0; adv[i]; i++ ) { + if ( bi->bi_attrs[i]->ai_desc != adv[i] ) { + for ( j = i+1; j < bi->bi_nattrs; j++ ) { + if ( bi->bi_attrs[j]->ai_desc == adv[i] ) { + AttrInfo *ai = bi->bi_attrs[i]; + bi->bi_attrs[i] = bi->bi_attrs[j]; + bi->bi_attrs[j] = ai; + break; + } + } + if ( j == bi->bi_nattrs ) { + Debug( LDAP_DEBUG_ANY, + LDAP_XSTRING(bdb_tool_entry_reindex) + ": no index configured for %s\n", + adv[i]->ad_cname.bv_val, 0, 0 ); + return -1; + } + } + } + bi->bi_nattrs = i; + } + + /* Get the first attribute to index */ + if (bi->bi_linear_index && !index_nattrs) { + index_nattrs = bi->bi_nattrs - 1; + bi->bi_nattrs = 1; + } + + e = bdb_tool_entry_get( be, id ); + + if( e == NULL ) { + Debug( LDAP_DEBUG_ANY, + LDAP_XSTRING(bdb_tool_entry_reindex) + ": could not locate id=%ld\n", + (long) id, 0, 0 ); + return -1; + } + + op.o_hdr = &ohdr; + op.o_bd = be; + op.o_tmpmemctx = NULL; + op.o_tmpmfuncs = &ch_mfuncs; + + if (! (slapMode & SLAP_TOOL_QUICK)) { + rc = TXN_BEGIN( bi->bi_dbenv, NULL, &tid, bi->bi_db_opflags ); + if( rc != 0 ) { + Debug( LDAP_DEBUG_ANY, + "=> " LDAP_XSTRING(bdb_tool_entry_reindex) ": " + "txn_begin failed: %s (%d)\n", + db_strerror(rc), rc, 0 ); + goto done; + } + Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(bdb_tool_entry_reindex) ": txn id: %x\n", + tid->id(tid), 0, 0 ); + } + + /* + * just (re)add them for now + * assume that some other routine (not yet implemented) + * will zap index databases + * + */ + + Debug( LDAP_DEBUG_TRACE, + "=> " LDAP_XSTRING(bdb_tool_entry_reindex) "( %ld, \"%s\" )\n", + (long) id, e->e_dn, 0 ); + + rc = bdb_tool_index_add( &op, tid, e ); + +done: + if( rc == 0 ) { + if (! (slapMode & SLAP_TOOL_QUICK)) { + rc = TXN_COMMIT( tid, 0 ); + if( rc != 0 ) { + Debug( LDAP_DEBUG_ANY, + "=> " LDAP_XSTRING(bdb_tool_entry_reindex) + ": txn_commit failed: %s (%d)\n", + db_strerror(rc), rc, 0 ); + e->e_id = NOID; + } + } + + } else { + if (! (slapMode & SLAP_TOOL_QUICK)) { + TXN_ABORT( tid ); + Debug( LDAP_DEBUG_ANY, + "=> " LDAP_XSTRING(bdb_tool_entry_reindex) + ": txn_aborted! %s (%d)\n", + db_strerror(rc), rc, 0 ); + } + e->e_id = NOID; + } + bdb_entry_release( &op, e, 0 ); + + return rc; +} + +ID bdb_tool_entry_modify( + BackendDB *be, + Entry *e, + struct berval *text ) +{ + int rc; + struct bdb_info *bdb; + DB_TXN *tid = NULL; + Operation op = {0}; + Opheader ohdr = {0}; + + assert( be != NULL ); + assert( slapMode & SLAP_TOOL_MODE ); + + assert( text != NULL ); + assert( text->bv_val != NULL ); + assert( text->bv_val[0] == '\0' ); /* overconservative? */ + + assert ( e->e_id != NOID ); + + Debug( LDAP_DEBUG_TRACE, + "=> " LDAP_XSTRING(bdb_tool_entry_modify) "( %ld, \"%s\" )\n", + (long) e->e_id, e->e_dn, 0 ); + + bdb = (struct bdb_info *) be->be_private; + + if (! (slapMode & SLAP_TOOL_QUICK)) { + if( cursor ) { + cursor->c_close( cursor ); + cursor = NULL; + } + rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid, + bdb->bi_db_opflags ); + if( rc != 0 ) { + snprintf( text->bv_val, text->bv_len, + "txn_begin failed: %s (%d)", + db_strerror(rc), rc ); + Debug( LDAP_DEBUG_ANY, + "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n", + text->bv_val, 0, 0 ); + return NOID; + } + Debug( LDAP_DEBUG_TRACE, LDAP_XSTRING(bdb_tool_entry_modify) ": txn id: %x\n", + tid->id(tid), 0, 0 ); + } + + op.o_hdr = &ohdr; + op.o_bd = be; + op.o_tmpmemctx = NULL; + op.o_tmpmfuncs = &ch_mfuncs; + + /* id2entry index */ + rc = bdb_id2entry_update( be, tid, e ); + if( rc != 0 ) { + snprintf( text->bv_val, text->bv_len, + "id2entry_add failed: %s (%d)", + db_strerror(rc), rc ); + Debug( LDAP_DEBUG_ANY, + "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n", + text->bv_val, 0, 0 ); + goto done; + } + +done: + if( rc == 0 ) { + if (! (slapMode & SLAP_TOOL_QUICK)) { + rc = TXN_COMMIT( tid, 0 ); + if( rc != 0 ) { + snprintf( text->bv_val, text->bv_len, + "txn_commit failed: %s (%d)", + db_strerror(rc), rc ); + Debug( LDAP_DEBUG_ANY, + "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": " + "%s\n", text->bv_val, 0, 0 ); + e->e_id = NOID; + } + } + + } else { + if (! (slapMode & SLAP_TOOL_QUICK)) { + TXN_ABORT( tid ); + snprintf( text->bv_val, text->bv_len, + "txn_aborted! %s (%d)", + db_strerror(rc), rc ); + Debug( LDAP_DEBUG_ANY, + "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n", + text->bv_val, 0, 0 ); + } + e->e_id = NOID; + } + + return e->e_id; +} + +#ifdef BDB_TOOL_IDL_CACHING +static int +bdb_tool_idl_cmp( const void *v1, const void *v2 ) +{ + const bdb_tool_idl_cache *c1 = v1, *c2 = v2; + int rc; + + if (( rc = c1->kstr.bv_len - c2->kstr.bv_len )) return rc; + return memcmp( c1->kstr.bv_val, c2->kstr.bv_val, c1->kstr.bv_len ); +} + +static int +bdb_tool_idl_flush_one( void *v1, void *arg ) +{ + bdb_tool_idl_cache *ic = v1; + DB *db = arg; + struct bdb_info *bdb = bdb_tool_info; + bdb_tool_idl_cache_entry *ice; + DBC *curs; + DBT key, data; + int i, rc; + ID id, nid; + + /* Freshly allocated, ignore it */ + if ( !ic->head && ic->count <= BDB_IDL_DB_SIZE ) { + return 0; + } + + rc = db->cursor( db, NULL, &curs, 0 ); + if ( rc ) + return -1; + + DBTzero( &key ); + DBTzero( &data ); + + bv2DBT( &ic->kstr, &key ); + + data.size = data.ulen = sizeof( ID ); + data.flags = DB_DBT_USERMEM; + data.data = &nid; + + rc = curs->c_get( curs, &key, &data, DB_SET ); + /* If key already exists and we're writing a range... */ + if ( rc == 0 && ic->count > BDB_IDL_DB_SIZE ) { + /* If it's not currently a range, must delete old info */ + if ( nid ) { + /* Skip lo */ + while ( curs->c_get( curs, &key, &data, DB_NEXT_DUP ) == 0 ) + curs->c_del( curs, 0 ); + + nid = 0; + /* Store range marker */ + curs->c_put( curs, &key, &data, DB_KEYFIRST ); + } else { + + /* Skip lo */ + rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP ); + + /* Get hi */ + rc = curs->c_get( curs, &key, &data, DB_NEXT_DUP ); + + /* Delete hi */ + curs->c_del( curs, 0 ); + } + BDB_ID2DISK( ic->last, &nid ); + curs->c_put( curs, &key, &data, DB_KEYLAST ); + rc = 0; + } else if ( rc && rc != DB_NOTFOUND ) { + rc = -1; + } else if ( ic->count > BDB_IDL_DB_SIZE ) { + /* range, didn't exist before */ + nid = 0; + rc = curs->c_put( curs, &key, &data, DB_KEYLAST ); + if ( rc == 0 ) { + BDB_ID2DISK( ic->first, &nid ); + rc = curs->c_put( curs, &key, &data, DB_KEYLAST ); + if ( rc == 0 ) { + BDB_ID2DISK( ic->last, &nid ); + rc = curs->c_put( curs, &key, &data, DB_KEYLAST ); + } + } + if ( rc ) { + rc = -1; + } + } else { + int n; + + /* Just a normal write */ + rc = 0; + for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) { + int end; + if ( ice->next ) { + end = IDBLOCK; + } else { + end = ic->count & (IDBLOCK-1); + if ( !end ) + end = IDBLOCK; + } + for ( i=0; i<end; i++ ) { + if ( !ice->ids[i] ) continue; + BDB_ID2DISK( ice->ids[i], &nid ); + rc = curs->c_put( curs, &key, &data, DB_NODUPDATA ); + if ( rc ) { + if ( rc == DB_KEYEXIST ) { + rc = 0; + continue; + } + rc = -1; + break; + } + } + if ( rc ) { + rc = -1; + break; + } + } + if ( ic->head ) { + ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock ); + ic->tail->next = bdb_tool_idl_free_list; + bdb_tool_idl_free_list = ic->head; + bdb->bi_idl_cache_size -= n; + ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock ); + } + } + if ( ic != db->app_private ) { + ch_free( ic ); + } else { + ic->head = ic->tail = NULL; + } + curs->c_close( curs ); + return rc; +} + +static int +bdb_tool_idl_flush_db( DB *db, bdb_tool_idl_cache *ic ) +{ + Avlnode *root = db->app_private; + int rc; + + db->app_private = ic; + rc = avl_apply( root, bdb_tool_idl_flush_one, db, -1, AVL_INORDER ); + avl_free( root, NULL ); + db->app_private = NULL; + if ( rc != -1 ) + rc = 0; + return rc; +} + +static int +bdb_tool_idl_flush( BackendDB *be ) +{ + struct bdb_info *bdb = (struct bdb_info *) be->be_private; + DB *db; + Avlnode *root; + int i, rc = 0; + + for ( i=BDB_NDB; i < bdb->bi_ndatabases; i++ ) { + db = bdb->bi_databases[i]->bdi_db; + if ( !db->app_private ) continue; + rc = bdb_tool_idl_flush_db( db, NULL ); + if ( rc ) + break; + } + if ( !rc ) { + bdb->bi_idl_cache_size = 0; + } + return rc; +} + +int bdb_tool_idl_add( + BackendDB *be, + DB *db, + DB_TXN *txn, + DBT *key, + ID id ) +{ + struct bdb_info *bdb = (struct bdb_info *) be->be_private; + bdb_tool_idl_cache *ic, itmp; + bdb_tool_idl_cache_entry *ice; + int rc; + + if ( !bdb->bi_idl_cache_max_size ) + return bdb_idl_insert_key( be, db, txn, key, id ); + + DBT2bv( key, &itmp.kstr ); + + ic = avl_find( (Avlnode *)db->app_private, &itmp, bdb_tool_idl_cmp ); + + /* No entry yet, create one */ + if ( !ic ) { + DBC *curs; + DBT data; + ID nid; + int rc; + + ic = ch_malloc( sizeof( bdb_tool_idl_cache ) + itmp.kstr.bv_len ); + ic->kstr.bv_len = itmp.kstr.bv_len; + ic->kstr.bv_val = (char *)(ic+1); + AC_MEMCPY( ic->kstr.bv_val, itmp.kstr.bv_val, ic->kstr.bv_len ); + ic->head = ic->tail = NULL; + ic->last = 0; + ic->count = 0; + avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp, + avl_dup_error ); + + /* load existing key count here */ + rc = db->cursor( db, NULL, &curs, 0 ); + if ( rc ) return rc; + + data.ulen = sizeof( ID ); + data.flags = DB_DBT_USERMEM; + data.data = &nid; + rc = curs->c_get( curs, key, &data, DB_SET ); + if ( rc == 0 ) { + if ( nid == 0 ) { + ic->count = BDB_IDL_DB_SIZE+1; + } else { + db_recno_t count; + + curs->c_count( curs, &count, 0 ); + ic->count = count; + BDB_DISK2ID( &nid, &ic->first ); + } + } + curs->c_close( curs ); + } + /* are we a range already? */ + if ( ic->count > BDB_IDL_DB_SIZE ) { + ic->last = id; + return 0; + /* Are we at the limit, and converting to a range? */ + } else if ( ic->count == BDB_IDL_DB_SIZE ) { + int n; + for ( ice = ic->head, n=0; ice; ice = ice->next, n++ ) + /* counting */ ; + if ( n ) { + ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock ); + ic->tail->next = bdb_tool_idl_free_list; + bdb_tool_idl_free_list = ic->head; + bdb->bi_idl_cache_size -= n; + ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock ); + } + ic->head = ic->tail = NULL; + ic->last = id; + ic->count++; + return 0; + } + /* No free block, create that too */ + if ( !ic->tail || ( ic->count & (IDBLOCK-1)) == 0) { + ice = NULL; + ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock ); + if ( bdb->bi_idl_cache_size >= bdb->bi_idl_cache_max_size ) { + ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock ); + rc = bdb_tool_idl_flush_db( db, ic ); + if ( rc ) + return rc; + avl_insert( (Avlnode **)&db->app_private, ic, bdb_tool_idl_cmp, + avl_dup_error ); + ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_lrulock ); + } + bdb->bi_idl_cache_size++; + if ( bdb_tool_idl_free_list ) { + ice = bdb_tool_idl_free_list; + bdb_tool_idl_free_list = ice->next; + } + ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_lrulock ); + if ( !ice ) { + ice = ch_malloc( sizeof( bdb_tool_idl_cache_entry )); + } + memset( ice, 0, sizeof( *ice )); + if ( !ic->head ) { + ic->head = ice; + } else { + ic->tail->next = ice; + } + ic->tail = ice; + if ( !ic->count ) + ic->first = id; + } + ice = ic->tail; + ice->ids[ ic->count & (IDBLOCK-1) ] = id; + ic->count++; + + return 0; +} +#endif + +#ifdef USE_TRICKLE +static void * +bdb_tool_trickle_task( void *ctx, void *ptr ) +{ + DB_ENV *env = ptr; + int wrote; + + ldap_pvt_thread_mutex_lock( &bdb_tool_trickle_mutex ); + bdb_tool_trickle_active = 1; + ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond_end ); + while ( 1 ) { + ldap_pvt_thread_cond_wait( &bdb_tool_trickle_cond, + &bdb_tool_trickle_mutex ); + if ( slapd_shutdown ) + break; + env->memp_trickle( env, 30, &wrote ); + } + bdb_tool_trickle_active = 0; + ldap_pvt_thread_cond_signal( &bdb_tool_trickle_cond_end ); + ldap_pvt_thread_mutex_unlock( &bdb_tool_trickle_mutex ); + + return NULL; +} +#endif + +static void * +bdb_tool_index_task( void *ctx, void *ptr ) +{ + int base = *(int *)ptr; + + free( ptr ); + while ( 1 ) { + ldap_pvt_thread_mutex_lock( &bdb_tool_index_mutex ); + bdb_tool_index_tcount--; + if ( !bdb_tool_index_tcount ) + ldap_pvt_thread_cond_signal( &bdb_tool_index_cond_main ); + ldap_pvt_thread_cond_wait( &bdb_tool_index_cond_work, + &bdb_tool_index_mutex ); + if ( slapd_shutdown ) { + bdb_tool_index_tcount--; + if ( !bdb_tool_index_tcount ) + ldap_pvt_thread_cond_signal( &bdb_tool_index_cond_main ); + ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex ); + break; + } + ldap_pvt_thread_mutex_unlock( &bdb_tool_index_mutex ); + + bdb_tool_index_threads[base] = bdb_index_recrun( bdb_tool_ix_op, + bdb_tool_info, bdb_tool_index_rec, bdb_tool_ix_id, base ); + } + + return NULL; +} |