Adding upstream version 2:4.17.12+dfsg.upstream/2%4.17.12+dfsgupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

15 files changed, 14765 insertions, 0 deletions

diff --git a/lib/ldb/common/attrib_handlers.c b/lib/ldb/common/attrib_handlers.c
new file mode 100644
index 0000000..febf2f4
--- /dev/null
+++ b/lib/ldb/common/attrib_handlers.c
@@ -0,0 +1,634 @@
+/* 
+   ldb database library
+
+   Copyright (C) Andrew Tridgell  2005
+   Copyright (C) Andrew Bartlett <abartlet@samba.org> 2006-2009
+
+     ** NOTE! The following LGPL license applies to the ldb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+   
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+/*
+  attribute handlers for well known attribute types, selected by syntax OID
+  see rfc2252
+*/
+
+#include "ldb_private.h"
+#include "system/locale.h"
+#include "ldb_handlers.h"
+
+/*
+  default handler that just copies a ldb_val.
+*/
+int ldb_handler_copy(struct ldb_context *ldb, void *mem_ctx,
+		     const struct ldb_val *in, struct ldb_val *out)
+{
+	*out = ldb_val_dup(mem_ctx, in);
+	if (in->length > 0 && out->data == NULL) {
+		ldb_oom(ldb);
+		return -1;
+	}
+	return 0;
+}
+
+/*
+  a case folding copy handler, removing leading and trailing spaces and
+  multiple internal spaces
+
+  We exploit the fact that utf8 never uses the space octet except for
+  the space itself
+*/
+int ldb_handler_fold(struct ldb_context *ldb, void *mem_ctx,
+			    const struct ldb_val *in, struct ldb_val *out)
+{
+	char *s, *t, *start;
+	bool in_space;
+
+	if (!in || !out || !(in->data)) {
+		return -1;
+	}
+
+	out->data = (uint8_t *)ldb_casefold(ldb, mem_ctx, (const char *)(in->data), in->length);
+	if (out->data == NULL) {
+		ldb_debug(ldb, LDB_DEBUG_ERROR, "ldb_handler_fold: unable to casefold string [%.*s]", (int)in->length, (const char *)in->data);
+		return -1;
+	}
+
+	start = (char *)(out->data);
+	in_space = true;
+	t = start;
+	for (s = start; *s != '\0'; s++) {
+		if (*s == ' ') {
+			if (in_space) {
+				/*
+				 * We already have one (or this is the start)
+				 * and we don't want to add more
+				 */
+				continue;
+			}
+			in_space = true;
+		} else {
+			in_space = false;
+		}
+		*t = *s;
+		t++;
+	}
+
+	if (in_space && t != start) {
+		/* the loop will have left a single trailing space */
+		t--;
+	}
+	*t = '\0';
+
+	out->length = t - start;
+	return 0;
+}
+
+/* length limited conversion of a ldb_val to an int64_t */
+static int val_to_int64(const struct ldb_val *in, int64_t *v)
+{
+	char *end;
+	char buf[64];
+
+	/* make sure we don't read past the end of the data */
+	if (in->length > sizeof(buf)-1) {
+		return LDB_ERR_INVALID_ATTRIBUTE_SYNTAX;
+	}
+	strncpy(buf, (char *)in->data, in->length);
+	buf[in->length] = 0;
+
+	*v = (int64_t) strtoll(buf, &end, 0);
+	if (*end != 0) {
+		return LDB_ERR_INVALID_ATTRIBUTE_SYNTAX;
+	}
+	return LDB_SUCCESS;
+}
+
+
+/*
+  canonicalise a ldap Integer
+  rfc2252 specifies it should be in decimal form
+*/
+static int ldb_canonicalise_Integer(struct ldb_context *ldb, void *mem_ctx,
+				    const struct ldb_val *in, struct ldb_val *out)
+{
+	int64_t i;
+	int ret;
+
+	ret = val_to_int64(in, &i);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+	out->data = (uint8_t *) talloc_asprintf(mem_ctx, "%lld", (long long)i);
+	if (out->data == NULL) {
+		ldb_oom(ldb);
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	out->length = strlen((char *)out->data);
+	return 0;
+}
+
+/*
+ * Lexicographically ordered format for a ldap Integer
+ *
+ * [ INT64_MIN ... -3, -2, -1 | 0 | +1, +2, +3 ... INT64_MAX ]
+ *             n                o              p
+ *
+ * For human readability sake, we continue to format the key as a string
+ * (like the canonicalize) rather than store as a fixed binary representation.
+ *
+ * In order to sort the integers in the correct string order, there are three
+ * techniques we use:
+ *
+ * 1. Zero padding
+ * 2. Negative integer inversion
+ * 3. 1-byte prefixes: 'n' < 'o' < 'p'
+ *
+ * 1. To have a fixed-width representation so that 10 sorts after 2 rather than
+ * after 1, we zero pad, like this 4-byte width example:
+ *
+ *     0001, 0002, 0010
+ *
+ * INT64_MAX = 2^63 - 1 = 9223372036854775807 (19 characters long)
+ *
+ * Meaning we need to pad to 19 characters.
+ *
+ * 2. This works for positive integers, but negative integers will still be
+ * sorted backwards, for example:
+ *
+ *     -9223372036854775808 ..., -0000000000000000002, -0000000000000000001
+ *          INT64_MIN                    -2                    -1
+ *
+ *   gets sorted based on string as:
+ *
+ *     -0000000000000000001, -0000000000000000002, ... -9223372036854775808
+ *
+ * In order to fix this, we invert the negative integer range, so that they
+ * get sorted the same way as positive numbers. INT64_MIN becomes the lowest
+ * possible non-negative number (zero), and -1 becomes the highest (INT64_MAX).
+ *
+ * The actual conversion applied to negative number 'x' is:
+ *   INT64_MAX - abs(x) + 1
+ * (The +1 is needed because abs(INT64_MIN) is one greater than INT64_MAX)
+ *
+ * 3. Finally, we now have two different numbers that map to the same key, e.g.
+ * INT64_MIN maps to -0000000000000000000 and zero maps to 0000000000000000000.
+ * In order to avoid confusion, we give every number a prefix representing its
+ * sign: 'n' for negative numbers, 'o' for zero, and 'p' for positive. (Note
+ * that '+' and '-' weren't used because they sort the wrong way).
+ *
+ * The result is a range of key values that look like this:
+ *
+ *     n0000000000000000000, ... n9223372036854775807,
+ *          INT64_MIN                    -1
+ *
+ *     o0000000000000000000,
+ *            ZERO
+ *
+ *     p0000000000000000001, ... p9223372036854775807
+ *            +1                       INT64_MAX
+ */
+static int ldb_index_format_Integer(struct ldb_context *ldb,
+				    void *mem_ctx,
+				    const struct ldb_val *in,
+				    struct ldb_val *out)
+{
+	int64_t i;
+	int ret;
+	char prefix;
+	size_t len;
+
+	ret = val_to_int64(in, &i);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	if (i < 0) {
+		/*
+		 * i is negative, so this is subtraction rather than
+		 * wrap-around.
+		 */
+		prefix = 'n';
+		i = INT64_MAX + i + 1;
+	} else if (i > 0) {
+		prefix = 'p';
+	} else {
+		prefix = 'o';
+	}
+
+	out->data = (uint8_t *) talloc_asprintf(mem_ctx, "%c%019lld", prefix, (long long)i);
+	if (out->data == NULL) {
+		ldb_oom(ldb);
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	len = talloc_array_length(out->data) - 1;
+	if (len != 20) {
+		ldb_debug(ldb, LDB_DEBUG_ERROR,
+			  __location__ ": expected index format str %s to"
+			  " have length 20 but got %zu",
+			  (char*)out->data, len);
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	out->length = 20;
+	return 0;
+}
+
+/*
+  compare two Integers
+*/
+static int ldb_comparison_Integer(struct ldb_context *ldb, void *mem_ctx,
+				  const struct ldb_val *v1, const struct ldb_val *v2)
+{
+	int64_t i1=0, i2=0;
+	val_to_int64(v1, &i1);
+	val_to_int64(v2, &i2);
+	if (i1 == i2) return 0;
+	return i1 > i2? 1 : -1;
+}
+
+/*
+  canonicalise a ldap Boolean
+  rfc2252 specifies it should be either "TRUE" or "FALSE"
+*/
+static int ldb_canonicalise_Boolean(struct ldb_context *ldb, void *mem_ctx,
+			     const struct ldb_val *in, struct ldb_val *out)
+{
+	if (in->length >= 4 && strncasecmp((char *)in->data, "TRUE", in->length) == 0) {
+		out->data = (uint8_t *)talloc_strdup(mem_ctx, "TRUE");
+		out->length = 4;
+	} else if (in->length >= 5 && strncasecmp((char *)in->data, "FALSE", in->length) == 0) {
+		out->data = (uint8_t *)talloc_strdup(mem_ctx, "FALSE");
+		out->length = 5;
+	} else {
+		return -1;
+	}
+	return 0;
+}
+
+/*
+  compare two Booleans
+*/
+static int ldb_comparison_Boolean(struct ldb_context *ldb, void *mem_ctx,
+			   const struct ldb_val *v1, const struct ldb_val *v2)
+{
+	if (v1->length != v2->length) {
+		return v1->length - v2->length;
+	}
+	return strncasecmp((char *)v1->data, (char *)v2->data, v1->length);
+}
+
+
+/*
+  compare two binary blobs
+*/
+int ldb_comparison_binary(struct ldb_context *ldb, void *mem_ctx,
+			  const struct ldb_val *v1, const struct ldb_val *v2)
+{
+	if (v1->length != v2->length) {
+		return v1->length - v2->length;
+	}
+	return memcmp(v1->data, v2->data, v1->length);
+}
+
+/*
+  compare two case insensitive strings, ignoring multiple whitespaces
+  and leading and trailing whitespaces
+  see rfc2252 section 8.1
+	
+  try to optimize for the ascii case,
+  but if we find out an utf8 codepoint revert to slower but correct function
+*/
+int ldb_comparison_fold(struct ldb_context *ldb, void *mem_ctx,
+			       const struct ldb_val *v1, const struct ldb_val *v2)
+{
+	const char *s1=(const char *)v1->data, *s2=(const char *)v2->data;
+	size_t n1 = v1->length, n2 = v2->length;
+	char *b1, *b2;
+	const char *u1, *u2;
+	int ret;
+	while (n1 && *s1 == ' ') { s1++; n1--; };
+	while (n2 && *s2 == ' ') { s2++; n2--; };
+
+	while (n1 && n2 && *s1 && *s2) {
+		/* the first 127 (0x7F) chars are ascii and utf8 guarantes they
+		 * never appear in multibyte sequences */
+		if (((unsigned char)s1[0]) & 0x80) goto utf8str;
+		if (((unsigned char)s2[0]) & 0x80) goto utf8str;
+		if (toupper((unsigned char)*s1) != toupper((unsigned char)*s2))
+			break;
+		if (*s1 == ' ') {
+			while (n1 > 1 && s1[0] == s1[1]) { s1++; n1--; }
+			while (n2 > 1 && s2[0] == s2[1]) { s2++; n2--; }
+		}
+		s1++; s2++;
+		n1--; n2--;
+	}
+
+	/* check for trailing spaces only if the other pointers has
+	 * reached the end of the strings otherwise we can
+	 * mistakenly match.  ex. "domain users" <->
+	 * "domainUpdates"
+	 */
+	if (n1 && *s1 == ' ' && (!n2 || !*s2)) {
+		while (n1 && *s1 == ' ') { s1++; n1--; }		
+	}
+	if (n2 && *s2 == ' ' && (!n1 || !*s1)) {
+		while (n2 && *s2 == ' ') { s2++; n2--; }		
+	}
+	if (n1 == 0 && n2 != 0) {
+		return -(int)toupper(*s2);
+	}
+	if (n2 == 0 && n1 != 0) {
+		return (int)toupper(*s1);
+	}
+	if (n1 == 0 && n2 == 0) {
+		return 0;
+	}
+	return (int)toupper(*s1) - (int)toupper(*s2);
+
+utf8str:
+	/* no need to recheck from the start, just from the first utf8 char found */
+	b1 = ldb_casefold(ldb, mem_ctx, s1, n1);
+	b2 = ldb_casefold(ldb, mem_ctx, s2, n2);
+
+	if (!b1 || !b2) {
+		/* One of the strings was not UTF8, so we have no
+		 * options but to do a binary compare */
+		talloc_free(b1);
+		talloc_free(b2);
+		ret = memcmp(s1, s2, MIN(n1, n2));
+		if (ret == 0) {
+			if (n1 == n2) return 0;
+			if (n1 > n2) {
+				return (int)toupper(s1[n2]);
+			} else {
+				return -(int)toupper(s2[n1]);
+			}
+		}
+		return ret;
+	}
+
+	u1 = b1;
+	u2 = b2;
+
+	while (*u1 & *u2) {
+		if (*u1 != *u2)
+			break;
+		if (*u1 == ' ') {
+			while (u1[0] == u1[1]) u1++;
+			while (u2[0] == u2[1]) u2++;
+		}
+		u1++; u2++;
+	}
+	if (! (*u1 && *u2)) {
+		while (*u1 == ' ') u1++;
+		while (*u2 == ' ') u2++;
+	}
+	ret = (int)(*u1 - *u2);
+
+	talloc_free(b1);
+	talloc_free(b2);
+	
+	return ret;
+}
+
+
+/*
+  canonicalise a attribute in DN format
+*/
+static int ldb_canonicalise_dn(struct ldb_context *ldb, void *mem_ctx,
+			       const struct ldb_val *in, struct ldb_val *out)
+{
+	struct ldb_dn *dn;
+	int ret = -1;
+
+	out->length = 0;
+	out->data = NULL;
+
+	dn = ldb_dn_from_ldb_val(mem_ctx, ldb, in);
+	if ( ! ldb_dn_validate(dn)) {
+		return LDB_ERR_INVALID_DN_SYNTAX;
+	}
+
+	out->data = (uint8_t *)ldb_dn_alloc_casefold(mem_ctx, dn);
+	if (out->data == NULL) {
+		goto done;
+	}
+	out->length = strlen((char *)out->data);
+
+	ret = 0;
+
+done:
+	talloc_free(dn);
+
+	return ret;
+}
+
+/*
+  compare two dns
+*/
+static int ldb_comparison_dn(struct ldb_context *ldb, void *mem_ctx,
+			     const struct ldb_val *v1, const struct ldb_val *v2)
+{
+	struct ldb_dn *dn1 = NULL, *dn2 = NULL;
+	int ret;
+
+	dn1 = ldb_dn_from_ldb_val(mem_ctx, ldb, v1);
+	if ( ! ldb_dn_validate(dn1)) return -1;
+
+	dn2 = ldb_dn_from_ldb_val(mem_ctx, ldb, v2);
+	if ( ! ldb_dn_validate(dn2)) {
+		talloc_free(dn1);
+		return -1;
+	} 
+
+	ret = ldb_dn_compare(dn1, dn2);
+
+	talloc_free(dn1);
+	talloc_free(dn2);
+	return ret;
+}
+
+/*
+  compare two utc time values. 1 second resolution
+*/
+static int ldb_comparison_utctime(struct ldb_context *ldb, void *mem_ctx,
+				  const struct ldb_val *v1, const struct ldb_val *v2)
+{
+	time_t t1=0, t2=0;
+	ldb_val_to_time(v1, &t1);
+	ldb_val_to_time(v2, &t2);
+	if (t1 == t2) return 0;
+	return t1 > t2? 1 : -1;
+}
+
+/*
+  canonicalise a utc time
+*/
+static int ldb_canonicalise_utctime(struct ldb_context *ldb, void *mem_ctx,
+				    const struct ldb_val *in, struct ldb_val *out)
+{
+	time_t t;
+	int ret;
+	ret = ldb_val_to_time(in, &t);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+	out->data = (uint8_t *)ldb_timestring_utc(mem_ctx, t);
+	if (out->data == NULL) {
+		ldb_oom(ldb);
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	out->length = strlen((char *)out->data);
+	return 0;
+}
+
+/*
+  canonicalise a generalized time
+*/
+static int ldb_canonicalise_generalizedtime(struct ldb_context *ldb, void *mem_ctx,
+				        const struct ldb_val *in, struct ldb_val *out)
+{
+	time_t t;
+	int ret;
+	ret = ldb_val_to_time(in, &t);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+	out->data = (uint8_t *)ldb_timestring(mem_ctx, t);
+	if (out->data == NULL) {
+		ldb_oom(ldb);
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	out->length = strlen((char *)out->data);
+	return 0;
+}
+
+/*
+  table of standard attribute handlers
+*/
+static const struct ldb_schema_syntax ldb_standard_syntaxes[] = {
+	{ 
+		.name            = LDB_SYNTAX_INTEGER,
+		.ldif_read_fn    = ldb_handler_copy,
+		.ldif_write_fn   = ldb_handler_copy,
+		.canonicalise_fn = ldb_canonicalise_Integer,
+		.comparison_fn   = ldb_comparison_Integer
+	},
+	{
+		.name            = LDB_SYNTAX_ORDERED_INTEGER,
+		.ldif_read_fn    = ldb_handler_copy,
+		.ldif_write_fn   = ldb_handler_copy,
+		.canonicalise_fn = ldb_canonicalise_Integer,
+		.index_format_fn = ldb_index_format_Integer,
+		.comparison_fn   = ldb_comparison_Integer
+	},
+	{
+		.name            = LDB_SYNTAX_OCTET_STRING,
+		.ldif_read_fn    = ldb_handler_copy,
+		.ldif_write_fn   = ldb_handler_copy,
+		.canonicalise_fn = ldb_handler_copy,
+		.comparison_fn   = ldb_comparison_binary
+	},
+	{ 
+		.name            = LDB_SYNTAX_DIRECTORY_STRING,
+		.ldif_read_fn    = ldb_handler_copy,
+		.ldif_write_fn   = ldb_handler_copy,
+		.canonicalise_fn = ldb_handler_fold,
+		.comparison_fn   = ldb_comparison_fold
+	},
+	{ 
+		.name            = LDB_SYNTAX_DN,
+		.ldif_read_fn    = ldb_handler_copy,
+		.ldif_write_fn   = ldb_handler_copy,
+		.canonicalise_fn = ldb_canonicalise_dn,
+		.comparison_fn   = ldb_comparison_dn
+	},
+	{ 
+		.name            = LDB_SYNTAX_OBJECTCLASS,
+		.ldif_read_fn    = ldb_handler_copy,
+		.ldif_write_fn   = ldb_handler_copy,
+		.canonicalise_fn = ldb_handler_fold,
+		.comparison_fn   = ldb_comparison_fold
+	},
+	{ 
+		.name            = LDB_SYNTAX_UTC_TIME,
+		.ldif_read_fn    = ldb_handler_copy,
+		.ldif_write_fn   = ldb_handler_copy,
+		.canonicalise_fn = ldb_canonicalise_utctime,
+		.comparison_fn   = ldb_comparison_utctime
+	},
+	{ 
+		.name            = LDB_SYNTAX_GENERALIZED_TIME,
+		.ldif_read_fn    = ldb_handler_copy,
+		.ldif_write_fn   = ldb_handler_copy,
+		.canonicalise_fn = ldb_canonicalise_generalizedtime,
+		.comparison_fn   = ldb_comparison_utctime
+	},
+	{
+		.name            = LDB_SYNTAX_BOOLEAN,
+		.ldif_read_fn    = ldb_handler_copy,
+		.ldif_write_fn   = ldb_handler_copy,
+		.canonicalise_fn = ldb_canonicalise_Boolean,
+		.comparison_fn   = ldb_comparison_Boolean
+	},
+};
+
+
+/*
+  return the attribute handlers for a given syntax name
+*/
+const struct ldb_schema_syntax *ldb_standard_syntax_by_name(struct ldb_context *ldb,
+							    const char *syntax)
+{
+	unsigned int i;
+	unsigned num_handlers = sizeof(ldb_standard_syntaxes)/sizeof(ldb_standard_syntaxes[0]);
+	/* TODO: should be replaced with a binary search */
+	for (i=0;i<num_handlers;i++) {
+		if (strcmp(ldb_standard_syntaxes[i].name, syntax) == 0) {
+			return &ldb_standard_syntaxes[i];
+		}
+	}
+	return NULL;
+}
+
+int ldb_any_comparison(struct ldb_context *ldb, void *mem_ctx, 
+		       ldb_attr_handler_t canonicalise_fn, 
+		       const struct ldb_val *v1,
+		       const struct ldb_val *v2)
+{
+	int ret, ret1, ret2;
+	struct ldb_val v1_canon, v2_canon;
+	TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
+
+	/* I could try and bail if tmp_ctx was NULL, but what return
+	 * value would I use?
+	 *
+	 * It seems easier to continue on the NULL context 
+	 */
+	ret1 = canonicalise_fn(ldb, tmp_ctx, v1, &v1_canon);
+	ret2 = canonicalise_fn(ldb, tmp_ctx, v2, &v2_canon);
+
+	if (ret1 == LDB_SUCCESS && ret2 == LDB_SUCCESS) {
+		ret = ldb_comparison_binary(ldb, mem_ctx, &v1_canon, &v2_canon);
+	} else {
+		ret = ldb_comparison_binary(ldb, mem_ctx, v1, v2);
+	}
+	talloc_free(tmp_ctx);
+	return ret;
+}
diff --git a/lib/ldb/common/ldb.c b/lib/ldb/common/ldb.c
new file mode 100644
index 0000000..c2599a0
--- /dev/null
+++ b/lib/ldb/common/ldb.c
@@ -0,0 +1,2198 @@
+/*
+   ldb database library
+
+   Copyright (C) Andrew Tridgell  2004
+   Copyright (C) Simo Sorce  2005-2008
+
+     ** NOTE! The following LGPL license applies to the ldb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+/*
+ *  Name: ldb
+ *
+ *  Component: ldb core API
+ *
+ *  Description: core API routines interfacing to ldb backends
+ *
+ *  Author: Andrew Tridgell
+ */
+
+#define TEVENT_DEPRECATED 1
+#include "ldb_private.h"
+#include "ldb.h"
+
+static int ldb_context_destructor(void *ptr)
+{
+	struct ldb_context *ldb = talloc_get_type(ptr, struct ldb_context);
+
+	if (ldb->transaction_active) {
+		ldb_debug(ldb, LDB_DEBUG_FATAL,
+			  "A transaction is still active in ldb context [%p] on %s",
+			  ldb, (const char *)ldb_get_opaque(ldb, "ldb_url"));
+	}
+
+	return 0;
+}
+
+/*
+  this is used to catch debug messages from events
+*/
+static void ldb_tevent_debug(void *context, enum tevent_debug_level level,
+			     const char *fmt, va_list ap)  PRINTF_ATTRIBUTE(3,0);
+
+static void ldb_tevent_debug(void *context, enum tevent_debug_level level,
+			     const char *fmt, va_list ap)
+{
+	struct ldb_context *ldb = talloc_get_type(context, struct ldb_context);
+	enum ldb_debug_level ldb_level = LDB_DEBUG_FATAL;
+
+	switch (level) {
+	case TEVENT_DEBUG_FATAL:
+		ldb_level = LDB_DEBUG_FATAL;
+		break;
+	case TEVENT_DEBUG_ERROR:
+		ldb_level = LDB_DEBUG_ERROR;
+		break;
+	case TEVENT_DEBUG_WARNING:
+		ldb_level = LDB_DEBUG_WARNING;
+		break;
+	case TEVENT_DEBUG_TRACE:
+		ldb_level = LDB_DEBUG_TRACE;
+		break;
+	};
+
+	/* There isn't a tevent: prefix here because to add it means
+	 * actually printing the string, and most of the time we don't
+	 * want to show it */
+	ldb_vdebug(ldb, ldb_level, fmt, ap);
+}
+
+/*
+   initialise a ldb context
+   The mem_ctx is required
+   The event_ctx is required
+*/
+struct ldb_context *ldb_init(TALLOC_CTX *mem_ctx, struct tevent_context *ev_ctx)
+{
+	struct ldb_context *ldb;
+	int ret;
+	const char *modules_path = getenv("LDB_MODULES_PATH");
+
+	if (modules_path == NULL) {
+		modules_path = LDB_MODULESDIR;
+	}
+
+	ret = ldb_modules_load(modules_path, LDB_VERSION);
+	if (ret != LDB_SUCCESS) {
+		return NULL;
+	}
+
+	ldb = talloc_zero(mem_ctx, struct ldb_context);
+	if (ldb == NULL) {
+		return NULL;
+	}
+
+	/* A new event context so that callers who don't want ldb
+	 * operating on their global event context can work without
+	 * having to provide their own private one explicitly */
+	if (ev_ctx == NULL) {
+		ev_ctx = tevent_context_init(ldb);
+		if (ev_ctx == NULL) {
+			talloc_free(ldb);
+			return NULL;
+		}
+		tevent_set_debug(ev_ctx, ldb_tevent_debug, ldb);
+		tevent_loop_allow_nesting(ev_ctx);
+	}
+
+	ret = ldb_setup_wellknown_attributes(ldb);
+	if (ret != LDB_SUCCESS) {
+		talloc_free(ldb);
+		return NULL;
+	}
+
+	ldb_set_utf8_default(ldb);
+	ldb_set_create_perms(ldb, 0666);
+	ldb_set_modules_dir(ldb, LDB_MODULESDIR);
+	ldb_set_event_context(ldb, ev_ctx);
+	ret = ldb_register_extended_match_rules(ldb);
+	if (ret != LDB_SUCCESS) {
+		talloc_free(ldb);
+		return NULL;
+	}
+
+	/* TODO: get timeout from options if available there */
+	ldb->default_timeout = 300; /* set default to 5 minutes */
+
+	talloc_set_destructor((TALLOC_CTX *)ldb, ldb_context_destructor);
+
+	return ldb;
+}
+
+/*
+  try to autodetect a basedn if none specified. This fixes one of my
+  pet hates about ldapsearch, which is that you have to get a long,
+  complex basedn right to make any use of it.
+*/
+void ldb_set_default_dns(struct ldb_context *ldb)
+{
+	TALLOC_CTX *tmp_ctx;
+	int ret;
+	struct ldb_result *res;
+	struct ldb_dn *tmp_dn=NULL;
+	static const char *attrs[] = {
+		"rootDomainNamingContext",
+		"configurationNamingContext",
+		"schemaNamingContext",
+		"defaultNamingContext",
+		NULL
+	};
+
+	tmp_ctx = talloc_new(ldb);
+	ret = ldb_search(ldb, tmp_ctx, &res, ldb_dn_new(tmp_ctx, ldb, NULL),
+			 LDB_SCOPE_BASE, attrs, "(objectClass=*)");
+	if (ret != LDB_SUCCESS) {
+		talloc_free(tmp_ctx);
+		return;
+	}
+
+	if (res->count != 1) {
+		talloc_free(tmp_ctx);
+		return;
+	}
+
+	if (!ldb_get_opaque(ldb, "rootDomainNamingContext")) {
+		tmp_dn = ldb_msg_find_attr_as_dn(ldb, ldb, res->msgs[0],
+						 "rootDomainNamingContext");
+		ldb_set_opaque(ldb, "rootDomainNamingContext", tmp_dn);
+	}
+
+	if (!ldb_get_opaque(ldb, "configurationNamingContext")) {
+		tmp_dn = ldb_msg_find_attr_as_dn(ldb, ldb, res->msgs[0],
+						 "configurationNamingContext");
+		ldb_set_opaque(ldb, "configurationNamingContext", tmp_dn);
+	}
+
+	if (!ldb_get_opaque(ldb, "schemaNamingContext")) {
+		tmp_dn = ldb_msg_find_attr_as_dn(ldb, ldb, res->msgs[0],
+						 "schemaNamingContext");
+		ldb_set_opaque(ldb, "schemaNamingContext", tmp_dn);
+	}
+
+	if (!ldb_get_opaque(ldb, "defaultNamingContext")) {
+		tmp_dn = ldb_msg_find_attr_as_dn(ldb, ldb, res->msgs[0],
+						 "defaultNamingContext");
+		ldb_set_opaque(ldb, "defaultNamingContext", tmp_dn);
+	}
+
+	talloc_free(tmp_ctx);
+}
+
+struct ldb_dn *ldb_get_root_basedn(struct ldb_context *ldb)
+{
+	void *opaque = ldb_get_opaque(ldb, "rootDomainNamingContext");
+	return talloc_get_type(opaque, struct ldb_dn);
+}
+
+struct ldb_dn *ldb_get_config_basedn(struct ldb_context *ldb)
+{
+	void *opaque = ldb_get_opaque(ldb, "configurationNamingContext");
+	return talloc_get_type(opaque, struct ldb_dn);
+}
+
+struct ldb_dn *ldb_get_schema_basedn(struct ldb_context *ldb)
+{
+	void *opaque = ldb_get_opaque(ldb, "schemaNamingContext");
+	return talloc_get_type(opaque, struct ldb_dn);
+}
+
+struct ldb_dn *ldb_get_default_basedn(struct ldb_context *ldb)
+{
+	void *opaque = ldb_get_opaque(ldb, "defaultNamingContext");
+	return talloc_get_type(opaque, struct ldb_dn);
+}
+
+/*
+   connect to a database. The URL can either be one of the following forms
+   ldb://path
+   ldapi://path
+
+   flags is made up of LDB_FLG_*
+
+   the options are passed uninterpreted to the backend, and are
+   backend specific
+*/
+int ldb_connect(struct ldb_context *ldb, const char *url,
+		unsigned int flags, const char *options[])
+{
+	int ret;
+	char *url2;
+	/* We seem to need to do this here, or else some utilities don't
+	 * get ldb backends */
+
+	ldb->flags = flags;
+
+	url2 = talloc_strdup(ldb, url);
+	if (!url2) {
+		ldb_oom(ldb);
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	ret = ldb_set_opaque(ldb, "ldb_url", url2);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	/*
+	 * Take a copy of the options.
+	 */
+	ldb->options = ldb_options_copy(ldb, options);
+	if (ldb->options == NULL && options != NULL) {
+		ldb_oom(ldb);
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	ret = ldb_module_connect_backend(ldb, url, options, &ldb->modules);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	ret = ldb_load_modules(ldb, options);
+	if (ret != LDB_SUCCESS) {
+		ldb_debug(ldb, LDB_DEBUG_FATAL,
+			  "Unable to load modules for %s: %s",
+			  url, ldb_errstring(ldb));
+		return ret;
+	}
+
+	/* set the default base dn */
+	ldb_set_default_dns(ldb);
+
+	return LDB_SUCCESS;
+}
+
+void ldb_set_errstring(struct ldb_context *ldb, const char *err_string)
+{
+	ldb_asprintf_errstring(ldb, "%s", err_string);
+}
+
+void ldb_asprintf_errstring(struct ldb_context *ldb, const char *format, ...)
+{
+	va_list ap;
+	char *old_err_string = NULL;
+	if (ldb->err_string) {
+		old_err_string = ldb->err_string;
+	}
+
+	va_start(ap, format);
+	ldb->err_string = talloc_vasprintf(ldb, format, ap);
+	va_end(ap);
+
+	TALLOC_FREE(old_err_string);
+	
+	if (ldb->flags & LDB_FLG_ENABLE_TRACING) {
+		ldb_debug(ldb, LDB_DEBUG_TRACE, "ldb_asprintf/set_errstring: %s",
+			  ldb->err_string);
+	}
+}
+
+void ldb_reset_err_string(struct ldb_context *ldb)
+{
+	TALLOC_FREE(ldb->err_string);
+}
+
+
+
+/*
+  set an ldb error based on file:line
+*/
+int ldb_error_at(struct ldb_context *ldb, int ecode,
+		 const char *reason, const char *file, int line)
+{
+	if (reason == NULL) {
+		reason = ldb_strerror(ecode);
+	}
+	ldb_asprintf_errstring(ldb, "%s at %s:%d", reason, file, line);
+	return ecode;
+}
+
+
+#define FIRST_OP_NOERR(ldb, op) do { \
+	next_module = ldb->modules;					\
+	while (next_module && next_module->ops->op == NULL) {		\
+		next_module = next_module->next;			    \
+	};							    \
+	if ((ldb->flags & LDB_FLG_ENABLE_TRACING) && next_module) { \
+		ldb_debug(ldb, LDB_DEBUG_TRACE, "ldb_trace_request: (%s)->" #op, \
+			  next_module->ops->name);				\
+	}								\
+} while (0)
+
+#define FIRST_OP(ldb, op) do { \
+	FIRST_OP_NOERR(ldb, op); \
+	if (next_module == NULL) {	       				\
+		ldb_asprintf_errstring(ldb, "unable to find module or backend to handle operation: " #op); \
+		return LDB_ERR_OPERATIONS_ERROR;			\
+	} \
+} while (0)
+
+
+/*
+  start a transaction
+*/
+int ldb_transaction_start(struct ldb_context *ldb)
+{
+	struct ldb_module *next_module;
+	int status;
+
+	ldb_debug(ldb, LDB_DEBUG_TRACE,
+		  "start ldb transaction (nesting: %d)",
+		  ldb->transaction_active);
+
+	/* explicit transaction active, count nested requests */
+	if (ldb->transaction_active) {
+		ldb->transaction_active++;
+		return LDB_SUCCESS;
+	}
+
+	/* start a new transaction */
+	ldb->transaction_active++;
+	ldb->prepare_commit_done = false;
+
+	FIRST_OP(ldb, start_transaction);
+
+	ldb_reset_err_string(ldb);
+
+	status = next_module->ops->start_transaction(next_module);
+	if (status != LDB_SUCCESS) {
+		if (ldb->err_string == NULL) {
+			/* no error string was setup by the backend */
+			ldb_asprintf_errstring(ldb,
+				"ldb transaction start: %s (%d)",
+				ldb_strerror(status),
+				status);
+		ldb->transaction_active--;
+		}
+		if ((next_module && next_module->ldb->flags & LDB_FLG_ENABLE_TRACING)) {
+			ldb_debug(next_module->ldb, LDB_DEBUG_TRACE, "start ldb transaction error: %s",
+				  ldb_errstring(next_module->ldb));
+		}
+	} else {
+		if ((next_module && next_module->ldb->flags & LDB_FLG_ENABLE_TRACING)) {
+			ldb_debug(next_module->ldb, LDB_DEBUG_TRACE, "start ldb transaction success");
+		}
+	}
+	return status;
+}
+
+/*
+  prepare for transaction commit (first phase of two phase commit)
+*/
+int ldb_transaction_prepare_commit(struct ldb_context *ldb)
+{
+	struct ldb_module *next_module;
+	int status;
+
+	if (ldb->prepare_commit_done) {
+		return LDB_SUCCESS;
+	}
+
+	/* commit only when all nested transactions are complete */
+	if (ldb->transaction_active > 1) {
+		return LDB_SUCCESS;
+	}
+
+	ldb->prepare_commit_done = true;
+
+	if (ldb->transaction_active < 0) {
+		ldb_debug(ldb, LDB_DEBUG_FATAL,
+			  "prepare commit called but no ldb transactions are active!");
+		ldb->transaction_active = 0;
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	/* call prepare transaction if available */
+	FIRST_OP_NOERR(ldb, prepare_commit);
+	if (next_module == NULL) {
+		return LDB_SUCCESS;
+	}
+
+	ldb_reset_err_string(ldb);
+
+	status = next_module->ops->prepare_commit(next_module);
+	if (status != LDB_SUCCESS) {
+		ldb->transaction_active--;
+		/* if a next_module fails the prepare then we need
+		   to call the end transaction for everyone */
+		FIRST_OP(ldb, del_transaction);
+		next_module->ops->del_transaction(next_module);
+		if (ldb->err_string == NULL) {
+			/* no error string was setup by the backend */
+			ldb_asprintf_errstring(ldb,
+					       "ldb transaction prepare commit: %s (%d)",
+					       ldb_strerror(status),
+					       status);
+		}
+		if ((next_module && next_module->ldb->flags & LDB_FLG_ENABLE_TRACING)) {
+			ldb_debug(next_module->ldb, LDB_DEBUG_TRACE, "prepare commit transaction error: %s",
+				  ldb_errstring(next_module->ldb));
+		}
+	}
+
+	return status;
+}
+
+
+/*
+  commit a transaction
+*/
+int ldb_transaction_commit(struct ldb_context *ldb)
+{
+	struct ldb_module *next_module;
+	int status;
+
+	status = ldb_transaction_prepare_commit(ldb);
+	if (status != LDB_SUCCESS) {
+		return status;
+	}
+
+	ldb->transaction_active--;
+
+	ldb_debug(ldb, LDB_DEBUG_TRACE,
+		  "commit ldb transaction (nesting: %d)",
+		  ldb->transaction_active);
+
+	/* commit only when all nested transactions are complete */
+	if (ldb->transaction_active > 0) {
+		return LDB_SUCCESS;
+	}
+
+	if (ldb->transaction_active < 0) {
+		ldb_debug(ldb, LDB_DEBUG_FATAL,
+			  "commit called but no ldb transactions are active!");
+		ldb->transaction_active = 0;
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	ldb_reset_err_string(ldb);
+
+	FIRST_OP(ldb, end_transaction);
+	status = next_module->ops->end_transaction(next_module);
+	if (status != LDB_SUCCESS) {
+		if (ldb->err_string == NULL) {
+			/* no error string was setup by the backend */
+			ldb_asprintf_errstring(ldb,
+				"ldb transaction commit: %s (%d)",
+				ldb_strerror(status),
+				status);
+		}
+		if ((next_module && next_module->ldb->flags & LDB_FLG_ENABLE_TRACING)) {
+			ldb_debug(next_module->ldb, LDB_DEBUG_TRACE, "commit ldb transaction error: %s",
+				  ldb_errstring(next_module->ldb));
+		}
+	}
+	return status;
+}
+
+
+/*
+  cancel a transaction
+*/
+int ldb_transaction_cancel(struct ldb_context *ldb)
+{
+	struct ldb_module *next_module;
+	int status;
+
+	ldb->transaction_active--;
+
+	ldb_debug(ldb, LDB_DEBUG_TRACE,
+		  "cancel ldb transaction (nesting: %d)",
+		  ldb->transaction_active);
+
+	/* really cancel only if all nested transactions are complete */
+	if (ldb->transaction_active > 0) {
+		return LDB_SUCCESS;
+	}
+
+	if (ldb->transaction_active < 0) {
+		ldb_debug(ldb, LDB_DEBUG_FATAL,
+			  "cancel called but no ldb transactions are active!");
+		ldb->transaction_active = 0;
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	FIRST_OP(ldb, del_transaction);
+
+	status = next_module->ops->del_transaction(next_module);
+	if (status != LDB_SUCCESS) {
+		if (ldb->err_string == NULL) {
+			/* no error string was setup by the backend */
+			ldb_asprintf_errstring(ldb,
+				"ldb transaction cancel: %s (%d)",
+				ldb_strerror(status),
+				status);
+		}
+		if ((next_module && next_module->ldb->flags & LDB_FLG_ENABLE_TRACING)) {
+			ldb_debug(next_module->ldb, LDB_DEBUG_TRACE, "cancel ldb transaction error: %s",
+				  ldb_errstring(next_module->ldb));
+		}
+	}
+	return status;
+}
+
+/*
+  cancel a transaction with no error if no transaction is pending
+  used when we fork() to clear any parent transactions
+*/
+int ldb_transaction_cancel_noerr(struct ldb_context *ldb)
+{
+	if (ldb->transaction_active > 0) {
+		return ldb_transaction_cancel(ldb);
+	}
+	return LDB_SUCCESS;
+}
+
+
+/* autostarts a transaction if none active */
+static int ldb_autotransaction_request(struct ldb_context *ldb,
+				       struct ldb_request *req)
+{
+	int ret;
+
+	ret = ldb_transaction_start(ldb);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	ret = ldb_request(ldb, req);
+	if (ret == LDB_SUCCESS) {
+		ret = ldb_wait(req->handle, LDB_WAIT_ALL);
+	}
+
+	if (ret == LDB_SUCCESS) {
+		return ldb_transaction_commit(ldb);
+	}
+	ldb_transaction_cancel(ldb);
+
+	return ret;
+}
+
+int ldb_wait(struct ldb_handle *handle, enum ldb_wait_type type)
+{
+	struct tevent_context *ev;
+	int ret;
+
+	if (handle == NULL) {
+		return LDB_ERR_UNAVAILABLE;
+	}
+
+	if (handle->state == LDB_ASYNC_DONE) {
+		if ((handle->status != LDB_SUCCESS) &&
+		    (handle->ldb->err_string == NULL)) {
+			/* if no error string was setup by the backend */
+			ldb_asprintf_errstring(handle->ldb,
+					       "ldb_wait from %s with LDB_ASYNC_DONE: %s (%d)",
+					       handle->location,
+					       ldb_strerror(handle->status),
+					       handle->status);
+		}
+		return handle->status;
+	}
+
+	ev = ldb_handle_get_event_context(handle);
+	if (NULL == ev) {
+		return ldb_oom(handle->ldb);
+	}
+
+	switch (type) {
+	case LDB_WAIT_NONE:
+		ret = tevent_loop_once(ev);
+		if (ret != 0) {
+			return ldb_operr(handle->ldb);
+		}
+		if (handle->status == LDB_SUCCESS) {
+			return LDB_SUCCESS;
+		}
+		if (handle->ldb->err_string != NULL) {
+			return handle->status;
+		}
+		/*
+		 * if no error string was setup by the backend
+		 */
+		ldb_asprintf_errstring(handle->ldb,
+				       "ldb_wait from %s with LDB_WAIT_NONE: %s (%d)",
+				       handle->location,
+				       ldb_strerror(handle->status),
+				       handle->status);
+		return handle->status;
+
+	case LDB_WAIT_ALL:
+		while (handle->state != LDB_ASYNC_DONE) {
+			ret = tevent_loop_once(ev);
+			if (ret != 0) {
+				return ldb_operr(handle->ldb);
+			}
+			if (handle->status != LDB_SUCCESS) {
+				if (handle->ldb->err_string != NULL) {
+					return handle->status;
+				}
+				/*
+				 * if no error string was setup by the
+				 * backend
+				 */
+				ldb_asprintf_errstring(handle->ldb,
+						       "ldb_wait from %s with "
+						       "LDB_WAIT_ALL: %s (%d)",
+						       handle->location,
+						       ldb_strerror(handle->status),
+						       handle->status);
+				return handle->status;
+			}
+		}
+		if (handle->status == LDB_SUCCESS) {
+			return LDB_SUCCESS;
+		}
+		if (handle->ldb->err_string != NULL) {
+			return handle->status;
+		}
+		/*
+		 * if no error string was setup by the backend
+		 */
+		ldb_asprintf_errstring(handle->ldb,
+				       "ldb_wait from %s with LDB_WAIT_ALL,"
+				       " LDB_ASYNC_DONE: %s (%d)",
+				       handle->location,
+				       ldb_strerror(handle->status),
+				       handle->status);
+		return handle->status;
+	}
+
+	return LDB_SUCCESS;
+}
+
+/* set the specified timeout or, if timeout is 0 set the default timeout */
+int ldb_set_timeout(struct ldb_context *ldb,
+		    struct ldb_request *req,
+		    int timeout)
+{
+	if (req == NULL) return LDB_ERR_OPERATIONS_ERROR;
+
+	if (timeout != 0) {
+		req->timeout = timeout;
+	} else {
+		req->timeout = ldb->default_timeout;
+	}
+	req->starttime = time(NULL);
+
+	return LDB_SUCCESS;
+}
+
+/* calculates the new timeout based on the previous starttime and timeout */
+int ldb_set_timeout_from_prev_req(struct ldb_context *ldb,
+				  struct ldb_request *oldreq,
+				  struct ldb_request *newreq)
+{
+	if (newreq == NULL) return LDB_ERR_OPERATIONS_ERROR;
+
+	if (oldreq == NULL) {
+		return ldb_set_timeout(ldb, newreq, 0);
+	}
+
+	newreq->starttime = oldreq->starttime;
+	newreq->timeout = oldreq->timeout;
+
+	return LDB_SUCCESS;
+}
+
+
+struct ldb_handle *ldb_handle_new(TALLOC_CTX *mem_ctx, struct ldb_context *ldb)
+{
+	struct ldb_handle *h;
+
+	h = talloc_zero(mem_ctx, struct ldb_handle);
+	if (h == NULL) {
+		ldb_set_errstring(ldb, "Out of Memory");
+		return NULL;
+	}
+
+	h->status = LDB_SUCCESS;
+	h->state = LDB_ASYNC_INIT;
+	h->ldb = ldb;
+	h->flags = 0;
+	h->location = NULL;
+	h->parent = NULL;
+
+	if (h->ldb->require_private_event_context == true) {
+		h->event_context = tevent_context_init(h);
+		if (h->event_context == NULL) {
+			ldb_set_errstring(ldb,
+					  "Out of Memory allocating "
+					  "event context for new handle");
+			return NULL;
+		}
+		tevent_set_debug(h->event_context, ldb_tevent_debug, ldb);
+		tevent_loop_allow_nesting(h->event_context);
+	}
+
+	return h;
+}
+
+static struct ldb_handle *ldb_handle_new_child(TALLOC_CTX *mem_ctx,
+					       struct ldb_request *parent_req)
+{
+	struct ldb_handle *h;
+
+	h = talloc_zero(mem_ctx, struct ldb_handle);
+	if (h == NULL) {
+		ldb_set_errstring(parent_req->handle->ldb,
+				  "Out of Memory");
+		return NULL;
+	}
+
+	h->status = LDB_SUCCESS;
+	h->state = LDB_ASYNC_INIT;
+	h->ldb = parent_req->handle->ldb;
+	h->parent = parent_req;
+	h->nesting = parent_req->handle->nesting + 1;
+	h->flags = parent_req->handle->flags;
+	h->custom_flags = parent_req->handle->custom_flags;
+	h->event_context = parent_req->handle->event_context;
+
+	return h;
+}
+
+/*
+   set the permissions for new files to be passed to open() in
+   backends that use local files
+ */
+void ldb_set_create_perms(struct ldb_context *ldb, unsigned int perms)
+{
+	ldb->create_perms = perms;
+}
+
+unsigned int ldb_get_create_perms(struct ldb_context *ldb)
+{
+	return ldb->create_perms;
+}
+
+void ldb_set_event_context(struct ldb_context *ldb, struct tevent_context *ev)
+{
+	ldb->ev_ctx = ev;
+}
+
+struct tevent_context * ldb_get_event_context(struct ldb_context *ldb)
+{
+	return ldb->ev_ctx;
+}
+
+void ldb_request_set_state(struct ldb_request *req, int state)
+{
+	req->handle->state = state;
+}
+
+int ldb_request_get_status(struct ldb_request *req)
+{
+	return req->handle->status;
+}
+
+/*
+ * This function obtains the private event context for the handle,
+ * which may have been created to avoid nested event loops during
+ * ldb_tdb with the locks held
+ */
+struct tevent_context *ldb_handle_get_event_context(struct ldb_handle *handle)
+{
+	if (handle->event_context != NULL) {
+		return handle->event_context;
+	}
+	return ldb_get_event_context(handle->ldb);
+}
+
+/*
+ * This function forces a specific ldb handle to use the global event
+ * context.  This allows a nested event loop to operate, so any open
+ * transaction also needs to be aborted.
+ *
+ * Any events on this event context will be lost
+ *
+ * This is used in Samba when sending an IRPC to another part of the
+ * same process instead of making a local DB modification.
+ */
+void ldb_handle_use_global_event_context(struct ldb_handle *handle)
+{
+	TALLOC_FREE(handle->event_context);
+}
+
+void ldb_set_require_private_event_context(struct ldb_context *ldb)
+{
+	ldb->require_private_event_context = true;
+}
+
+/*
+  trace a ldb request
+*/
+static void ldb_trace_request(struct ldb_context *ldb, struct ldb_request *req)
+{
+	TALLOC_CTX *tmp_ctx = talloc_new(req);
+	unsigned int i;
+	struct ldb_ldif ldif;
+
+	switch (req->operation) {
+	case LDB_SEARCH:
+		ldb_debug_add(ldb, "ldb_trace_request: SEARCH\n");
+		ldb_debug_add(ldb, " dn: %s\n",
+			      ldb_dn_is_null(req->op.search.base)?"<rootDSE>":
+			      ldb_dn_get_linearized(req->op.search.base));
+		ldb_debug_add(ldb, " scope: %s\n", 
+			  req->op.search.scope==LDB_SCOPE_BASE?"base":
+			  req->op.search.scope==LDB_SCOPE_ONELEVEL?"one":
+			  req->op.search.scope==LDB_SCOPE_SUBTREE?"sub":"UNKNOWN");
+		ldb_debug_add(ldb, " expr: %s\n", 
+			  ldb_filter_from_tree(tmp_ctx, req->op.search.tree));
+		if (req->op.search.attrs == NULL) {
+			ldb_debug_add(ldb, " attr: <ALL>\n");
+		} else {
+			for (i=0; req->op.search.attrs[i]; i++) {
+				ldb_debug_add(ldb, " attr: %s\n", req->op.search.attrs[i]);
+			}
+		}
+		break;
+	case LDB_DELETE:
+		ldb_debug_add(ldb, "ldb_trace_request: DELETE\n");
+		ldb_debug_add(ldb, " dn: %s\n", 
+			      ldb_dn_get_linearized(req->op.del.dn));
+		break;
+	case LDB_RENAME:
+		ldb_debug_add(ldb, "ldb_trace_request: RENAME\n");
+		ldb_debug_add(ldb, " olddn: %s\n", 
+			      ldb_dn_get_linearized(req->op.rename.olddn));
+		ldb_debug_add(ldb, " newdn: %s\n", 
+			      ldb_dn_get_linearized(req->op.rename.newdn));
+		break;
+	case LDB_EXTENDED:
+		ldb_debug_add(ldb, "ldb_trace_request: EXTENDED\n");
+		ldb_debug_add(ldb, " oid: %s\n", req->op.extended.oid);
+		ldb_debug_add(ldb, " data: %s\n", req->op.extended.data?"yes":"no");
+		break;
+	case LDB_ADD:
+		ldif.changetype = LDB_CHANGETYPE_ADD;
+		ldif.msg = discard_const_p(struct ldb_message, req->op.add.message);
+
+		ldb_debug_add(ldb, "ldb_trace_request: ADD\n");
+
+		/* 
+		 * The choice to call
+		 * ldb_ldif_write_redacted_trace_string() is CRITICAL
+		 * for security.  It ensures that we do not output
+		 * passwords into debug logs 
+		 */
+
+		ldb_debug_add(req->handle->ldb, "%s\n", 
+			      ldb_ldif_write_redacted_trace_string(req->handle->ldb, tmp_ctx, &ldif));
+		break;
+	case LDB_MODIFY:
+		ldif.changetype = LDB_CHANGETYPE_MODIFY;
+		ldif.msg = discard_const_p(struct ldb_message, req->op.mod.message);
+
+		ldb_debug_add(ldb, "ldb_trace_request: MODIFY\n");
+
+		/* 
+		 * The choice to call
+		 * ldb_ldif_write_redacted_trace_string() is CRITICAL
+		 * for security.  It ensures that we do not output
+		 * passwords into debug logs 
+		 */
+
+		ldb_debug_add(req->handle->ldb, "%s\n", 
+			      ldb_ldif_write_redacted_trace_string(req->handle->ldb, tmp_ctx, &ldif));
+		break;
+	case LDB_REQ_REGISTER_CONTROL:
+		ldb_debug_add(ldb, "ldb_trace_request: REGISTER_CONTROL\n");
+		ldb_debug_add(req->handle->ldb, "%s\n", 
+			      req->op.reg_control.oid);
+		break;
+	case LDB_REQ_REGISTER_PARTITION:
+		ldb_debug_add(ldb, "ldb_trace_request: REGISTER_PARTITION\n");
+		ldb_debug_add(req->handle->ldb, "%s\n", 
+			      ldb_dn_get_linearized(req->op.reg_partition.dn));
+		break;
+	default:
+		ldb_debug_add(ldb, "ldb_trace_request: UNKNOWN(%u)\n", 
+			      req->operation);
+		break;
+	}
+
+	if (req->controls == NULL) {
+		ldb_debug_add(ldb, " control: <NONE>\n");
+	} else {
+		for (i=0; req->controls && req->controls[i]; i++) {
+			if (req->controls[i]->oid) {
+				ldb_debug_add(ldb, " control: %s  crit:%u  data:%s\n",
+					      req->controls[i]->oid,
+					      req->controls[i]->critical,
+					      req->controls[i]->data?"yes":"no");
+			}
+		}
+	}
+	
+	ldb_debug_end(ldb, LDB_DEBUG_TRACE);
+
+	talloc_free(tmp_ctx);
+}
+
+/*
+  check that the element flags don't have any internal bits set
+ */
+static int ldb_msg_check_element_flags(struct ldb_context *ldb,
+				       const struct ldb_message *message)
+{
+	unsigned i;
+	for (i=0; i<message->num_elements; i++) {
+		if (message->elements[i].flags & LDB_FLAG_INTERNAL_MASK) {
+			ldb_asprintf_errstring(ldb, "Invalid element flags 0x%08x on element %s in %s\n",
+					       message->elements[i].flags, message->elements[i].name,
+					       ldb_dn_get_linearized(message->dn));
+			return LDB_ERR_UNSUPPORTED_CRITICAL_EXTENSION;
+		}
+	}
+	return LDB_SUCCESS;
+}
+
+/*
+ * This context allows us to make the unlock be a talloc destructor
+ *
+ * This ensures that a request started, but not waited on, will still
+ * unlock.
+ */
+struct ldb_db_lock_context {
+	struct ldb_request *req;
+	struct ldb_context *ldb;
+};
+
+/*
+ * We have to have a the unlock on a destructor so that we unlock the
+ * DB if a caller calls talloc_free(req).  We trust that the ldb
+ * context has not already gone away.
+ */
+static int ldb_db_lock_destructor(struct ldb_db_lock_context *lock_context)
+{
+	int ret;
+	struct ldb_module *next_module;
+	FIRST_OP_NOERR(lock_context->ldb, read_unlock);
+	if (next_module != NULL) {
+		ret = next_module->ops->read_unlock(next_module);
+	} else {
+		ret = LDB_SUCCESS;
+	}
+
+	if (ret != LDB_SUCCESS) {
+		ldb_debug(lock_context->ldb,
+			  LDB_DEBUG_FATAL,
+			  "Failed to unlock db: %s / %s",
+			  ldb_errstring(lock_context->ldb),
+			  ldb_strerror(ret));
+	}
+	return 0;
+}
+
+static int ldb_lock_backend_callback(struct ldb_request *req,
+				     struct ldb_reply *ares)
+{
+	struct ldb_db_lock_context *lock_context;
+	int ret;
+
+	if (req->context == NULL) {
+		/*
+		 * The usual way to get here is to ignore the return codes
+		 * and continuing processing after an error.
+		 */
+		abort();
+	}
+	lock_context = talloc_get_type(req->context,
+				       struct ldb_db_lock_context);
+
+	if (!ares) {
+		return ldb_module_done(lock_context->req, NULL, NULL,
+					LDB_ERR_OPERATIONS_ERROR);
+	}
+	if (ares->error != LDB_SUCCESS || ares->type == LDB_REPLY_DONE) {
+		ret = ldb_module_done(lock_context->req, ares->controls,
+				      ares->response, ares->error);
+		/*
+		 * If this is a LDB_REPLY_DONE or an error, unlock the
+		 * DB by calling the destructor on this context
+		 */
+		TALLOC_FREE(req->context);
+		return ret;
+	}
+
+	/* Otherwise pass on the callback */
+	switch (ares->type) {
+	case LDB_REPLY_ENTRY:
+		return ldb_module_send_entry(lock_context->req, ares->message,
+					     ares->controls);
+
+	case LDB_REPLY_REFERRAL:
+		return ldb_module_send_referral(lock_context->req,
+						ares->referral);
+	default:
+		/* Can't happen */
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+}
+
+/*
+ * Do an ldb_search() with a lock held, but release it if the request
+ * is freed with talloc_free()
+ */
+static int lock_search(struct ldb_module *lock_module, struct ldb_request *req)
+{
+	/* Used in FIRST_OP_NOERR to find where to send the lock request */
+	struct ldb_module *next_module = NULL;
+	struct ldb_request *down_req = NULL;
+	struct ldb_db_lock_context *lock_context;
+	struct ldb_context *ldb = ldb_module_get_ctx(lock_module);
+	int ret;
+
+	lock_context = talloc(req, struct ldb_db_lock_context);
+	if (lock_context == NULL) {
+		return ldb_oom(ldb);
+	}
+
+	lock_context->ldb = ldb;
+	lock_context->req = req;
+
+	ret = ldb_build_search_req_ex(&down_req, ldb, req,
+				      req->op.search.base,
+				      req->op.search.scope,
+				      req->op.search.tree,
+				      req->op.search.attrs,
+				      req->controls,
+				      lock_context,
+				      ldb_lock_backend_callback,
+				      req);
+	LDB_REQ_SET_LOCATION(down_req);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	/* call DB lock */
+	FIRST_OP_NOERR(ldb, read_lock);
+	if (next_module != NULL) {
+		ret = next_module->ops->read_lock(next_module);
+	} else {
+		ret = LDB_ERR_UNSUPPORTED_CRITICAL_EXTENSION;
+	}
+
+	if (ret == LDB_ERR_UNSUPPORTED_CRITICAL_EXTENSION) {
+		/* We might be talking LDAP */
+		ldb_reset_err_string(ldb);
+		TALLOC_FREE(lock_context);
+
+		return ldb_next_request(lock_module, req);
+	} else if ((ret != LDB_SUCCESS) && (ldb->err_string == NULL)) {
+		/* if no error string was setup by the backend */
+		ldb_asprintf_errstring(ldb, "Failed to get DB lock: %s (%d)",
+				       ldb_strerror(ret), ret);
+	} else {
+		talloc_set_destructor(lock_context, ldb_db_lock_destructor);
+	}
+
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	return ldb_next_request(lock_module, down_req);
+}
+
+/*
+  start an ldb request
+  NOTE: the request must be a talloc context.
+  returns LDB_ERR_* on errors.
+*/
+int ldb_request(struct ldb_context *ldb, struct ldb_request *req)
+{
+	struct ldb_module *next_module;
+	int ret;
+
+	if (req->callback == NULL) {
+		ldb_set_errstring(ldb, "Requests MUST define callbacks");
+		return LDB_ERR_UNWILLING_TO_PERFORM;
+	}
+
+	ldb_reset_err_string(ldb);
+
+	if (ldb->flags & LDB_FLG_ENABLE_TRACING) {
+		ldb_trace_request(ldb, req);
+	}
+
+	/* call the first module in the chain */
+	switch (req->operation) {
+	case LDB_SEARCH:
+	{
+		/*
+		 * A fake module to allow ldb_next_request() to be
+		 * re-used and to keep the locking out of this function.
+		 */
+		static const struct ldb_module_ops lock_module_ops = {
+			.name = "lock_searches",
+			.search = lock_search
+		};
+		struct ldb_module lock_module = {
+			.ldb = ldb,
+			.next = ldb->modules,
+			.ops = &lock_module_ops
+		};
+		next_module = &lock_module;
+
+		/* due to "ldb_build_search_req" base DN always != NULL */
+		if (!ldb_dn_validate(req->op.search.base)) {
+			ldb_asprintf_errstring(ldb, "ldb_search: invalid basedn '%s'",
+					       ldb_dn_get_linearized(req->op.search.base));
+			return LDB_ERR_INVALID_DN_SYNTAX;
+		}
+
+		ret = next_module->ops->search(next_module, req);
+		break;
+	}
+	case LDB_ADD:
+		if (!ldb_dn_validate(req->op.add.message->dn)) {
+			ldb_asprintf_errstring(ldb, "ldb_add: invalid dn '%s'",
+					       ldb_dn_get_linearized(req->op.add.message->dn));
+			return LDB_ERR_INVALID_DN_SYNTAX;
+		}
+		/*
+		 * we have to normalize here, as so many places
+		 * in modules and backends assume we don't have two
+		 * elements with the same name
+		 */
+		ret = ldb_msg_normalize(ldb, req, req->op.add.message,
+		                        discard_const(&req->op.add.message));
+		if (ret != LDB_SUCCESS) {
+			ldb_oom(ldb);
+			return ret;
+		}
+		FIRST_OP(ldb, add);
+		ret = ldb_msg_check_element_flags(ldb, req->op.add.message);
+		if (ret != LDB_SUCCESS) {
+			/*
+			 * "ldb_msg_check_element_flags" generates an error
+			 * string
+			 */
+			return ret;
+		}
+		ret = next_module->ops->add(next_module, req);
+		break;
+	case LDB_MODIFY:
+		if (!ldb_dn_validate(req->op.mod.message->dn)) {
+			ldb_asprintf_errstring(ldb, "ldb_modify: invalid dn '%s'",
+					       ldb_dn_get_linearized(req->op.mod.message->dn));
+			return LDB_ERR_INVALID_DN_SYNTAX;
+		}
+		FIRST_OP(ldb, modify);
+		ret = ldb_msg_check_element_flags(ldb, req->op.mod.message);
+		if (ret != LDB_SUCCESS) {
+			/*
+			 * "ldb_msg_check_element_flags" generates an error
+			 * string
+			 */
+			return ret;
+		}
+		ret = next_module->ops->modify(next_module, req);
+		break;
+	case LDB_DELETE:
+		if (!ldb_dn_validate(req->op.del.dn)) {
+			ldb_asprintf_errstring(ldb, "ldb_delete: invalid dn '%s'",
+					       ldb_dn_get_linearized(req->op.del.dn));
+			return LDB_ERR_INVALID_DN_SYNTAX;
+		}
+		FIRST_OP(ldb, del);
+		ret = next_module->ops->del(next_module, req);
+		break;
+	case LDB_RENAME:
+		if (!ldb_dn_validate(req->op.rename.olddn)) {
+			ldb_asprintf_errstring(ldb, "ldb_rename: invalid olddn '%s'",
+					       ldb_dn_get_linearized(req->op.rename.olddn));
+			return LDB_ERR_INVALID_DN_SYNTAX;
+		}
+		if (!ldb_dn_validate(req->op.rename.newdn)) {
+			ldb_asprintf_errstring(ldb, "ldb_rename: invalid newdn '%s'",
+					       ldb_dn_get_linearized(req->op.rename.newdn));
+			return LDB_ERR_INVALID_DN_SYNTAX;
+		}
+		FIRST_OP(ldb, rename);
+		ret = next_module->ops->rename(next_module, req);
+		break;
+	case LDB_EXTENDED:
+		FIRST_OP(ldb, extended);
+		ret = next_module->ops->extended(next_module, req);
+		break;
+	default:
+		FIRST_OP(ldb, request);
+		ret = next_module->ops->request(next_module, req);
+		break;
+	}
+
+	if ((ret != LDB_SUCCESS) && (ldb->err_string == NULL)) {
+		/* if no error string was setup by the backend */
+		ldb_asprintf_errstring(ldb, "ldb_request: %s (%d)",
+				       ldb_strerror(ret), ret);
+	}
+
+	return ret;
+}
+
+int ldb_request_done(struct ldb_request *req, int status)
+{
+	req->handle->state = LDB_ASYNC_DONE;
+	req->handle->status = status;
+	return status;
+}
+
+/*
+  search the database given a LDAP-like search expression
+
+  returns an LDB error code
+
+  Use talloc_free to free the ldb_message returned in 'res', if successful
+
+*/
+int ldb_search_default_callback(struct ldb_request *req,
+				struct ldb_reply *ares)
+{
+	struct ldb_result *res;
+	unsigned int n;
+
+	res = talloc_get_type(req->context, struct ldb_result);
+
+	if (!ares) {
+		return ldb_request_done(req, LDB_ERR_OPERATIONS_ERROR);
+	}
+	if (ares->error != LDB_SUCCESS) {
+		return ldb_request_done(req, ares->error);
+	}
+
+	switch (ares->type) {
+	case LDB_REPLY_ENTRY:
+		res->msgs = talloc_realloc(res, res->msgs,
+					struct ldb_message *, res->count + 2);
+		if (! res->msgs) {
+			return ldb_request_done(req, LDB_ERR_OPERATIONS_ERROR);
+		}
+
+		res->msgs[res->count + 1] = NULL;
+
+		res->msgs[res->count] = talloc_move(res->msgs, &ares->message);
+		res->count++;
+		break;
+
+	case LDB_REPLY_REFERRAL:
+		if (res->refs) {
+			for (n = 0; res->refs[n]; n++) /*noop*/ ;
+		} else {
+			n = 0;
+		}
+
+		res->refs = talloc_realloc(res, res->refs, char *, n + 2);
+		if (! res->refs) {
+			return ldb_request_done(req, LDB_ERR_OPERATIONS_ERROR);
+		}
+
+		res->refs[n] = talloc_move(res->refs, &ares->referral);
+		res->refs[n + 1] = NULL;
+		break;
+
+	case LDB_REPLY_DONE:
+		/* TODO: we should really support controls on entries
+		 * and referrals too! */
+		res->controls = talloc_move(res, &ares->controls);
+
+		/* this is the last message, and means the request is done */
+		/* we have to signal and eventual ldb_wait() waiting that the
+		 * async request operation was completed */
+		talloc_free(ares);
+		return ldb_request_done(req, LDB_SUCCESS);
+	}
+
+	talloc_free(ares);
+
+	return LDB_SUCCESS;
+}
+
+int ldb_modify_default_callback(struct ldb_request *req, struct ldb_reply *ares)
+{
+	struct ldb_result *res;
+	unsigned int n;
+	int ret;
+
+	res = talloc_get_type(req->context, struct ldb_result);
+
+	if (!ares) {
+		return ldb_request_done(req, LDB_ERR_OPERATIONS_ERROR);
+	}
+
+	if (ares->error != LDB_SUCCESS) {
+		ret = ares->error;
+		talloc_free(ares);
+		return ldb_request_done(req, ret);
+	}
+
+	switch (ares->type) {
+	case LDB_REPLY_REFERRAL:
+		if (res->refs) {
+			for (n = 0; res->refs[n]; n++) /*noop*/ ;
+		} else {
+			n = 0;
+		}
+
+		res->refs = talloc_realloc(res, res->refs, char *, n + 2);
+		if (! res->refs) {
+			return ldb_request_done(req, LDB_ERR_OPERATIONS_ERROR);
+		}
+
+		res->refs[n] = talloc_move(res->refs, &ares->referral);
+		res->refs[n + 1] = NULL;
+		break;
+
+	case LDB_REPLY_DONE:
+		talloc_free(ares);
+		return ldb_request_done(req, LDB_SUCCESS);
+	default:
+		talloc_free(ares);
+		ldb_asprintf_errstring(req->handle->ldb, "Invalid LDB reply type %d", ares->type);
+		return ldb_request_done(req, LDB_ERR_OPERATIONS_ERROR);
+	}
+
+	talloc_free(ares);
+	return ldb_request_done(req, LDB_SUCCESS);
+}
+
+int ldb_op_default_callback(struct ldb_request *req, struct ldb_reply *ares)
+{
+	int ret;
+
+	if (!ares) {
+		return ldb_request_done(req, LDB_ERR_OPERATIONS_ERROR);
+	}
+
+	if (ares->error != LDB_SUCCESS) {
+		ret = ares->error;
+		talloc_free(ares);
+		return ldb_request_done(req, ret);
+	}
+
+	if (ares->type != LDB_REPLY_DONE) {
+		ldb_asprintf_errstring(req->handle->ldb, "Invalid LDB reply type %d", ares->type);
+		TALLOC_FREE(ares);
+		return ldb_request_done(req, LDB_ERR_OPERATIONS_ERROR);
+	}
+
+	talloc_free(ares);
+	return ldb_request_done(req, LDB_SUCCESS);
+}
+
+static struct ldb_request *ldb_build_req_common(TALLOC_CTX *mem_ctx,
+				struct ldb_context *ldb,
+				struct ldb_control **controls,
+				void *context,
+				ldb_request_callback_t callback,
+				struct ldb_request *parent)
+{
+	struct ldb_request *req = NULL;
+
+	req = talloc_zero(mem_ctx, struct ldb_request);
+	if (req == NULL) {
+		return NULL;
+	}
+	req->controls = controls;
+	req->context = context;
+	req->callback = callback;
+
+	ldb_set_timeout_from_prev_req(ldb, parent, req);
+
+	if (parent != NULL) {
+		req->handle = ldb_handle_new_child(req, parent);
+		if (req->handle == NULL) {
+			TALLOC_FREE(req);
+			return NULL;
+		}
+	} else {
+		req->handle = ldb_handle_new(req, ldb);
+		if (req->handle == NULL) {
+			TALLOC_FREE(req);
+			return NULL;
+		}
+	}
+
+	return req;
+}
+
+int ldb_build_search_req_ex(struct ldb_request **ret_req,
+			struct ldb_context *ldb,
+			TALLOC_CTX *mem_ctx,
+			struct ldb_dn *base,
+	       		enum ldb_scope scope,
+			struct ldb_parse_tree *tree,
+			const char * const *attrs,
+			struct ldb_control **controls,
+			void *context,
+			ldb_request_callback_t callback,
+			struct ldb_request *parent)
+{
+	struct ldb_request *req;
+
+	*ret_req = NULL;
+
+	req = ldb_build_req_common(mem_ctx, ldb, controls,
+				   context, callback, parent);
+	if (req == NULL) {
+		ldb_oom(ldb);
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	req->operation = LDB_SEARCH;
+	if (base == NULL) {
+		req->op.search.base = ldb_dn_new(req, ldb, NULL);
+	} else {
+		req->op.search.base = base;
+	}
+	req->op.search.scope = scope;
+
+	req->op.search.tree = tree;
+	if (req->op.search.tree == NULL) {
+		ldb_set_errstring(ldb, "'tree' can't be NULL");
+		talloc_free(req);
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	req->op.search.attrs = attrs;
+	*ret_req = req;
+	return LDB_SUCCESS;
+}
+
+int ldb_build_search_req(struct ldb_request **ret_req,
+			struct ldb_context *ldb,
+			TALLOC_CTX *mem_ctx,
+			struct ldb_dn *base,
+			enum ldb_scope scope,
+			const char *expression,
+			const char * const *attrs,
+			struct ldb_control **controls,
+			void *context,
+			ldb_request_callback_t callback,
+			struct ldb_request *parent)
+{
+	struct ldb_parse_tree *tree;
+	int ret;
+
+	tree = ldb_parse_tree(mem_ctx, expression);
+	if (tree == NULL) {
+		ldb_set_errstring(ldb, "Unable to parse search expression");
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	ret = ldb_build_search_req_ex(ret_req, ldb, mem_ctx, base,
+				      scope, tree, attrs, controls,
+				      context, callback, parent);
+	if (ret == LDB_SUCCESS) {
+		talloc_steal(*ret_req, tree);
+	}
+	return ret;
+}
+
+int ldb_build_add_req(struct ldb_request **ret_req,
+			struct ldb_context *ldb,
+			TALLOC_CTX *mem_ctx,
+			const struct ldb_message *message,
+			struct ldb_control **controls,
+			void *context,
+			ldb_request_callback_t callback,
+			struct ldb_request *parent)
+{
+	struct ldb_request *req;
+
+	*ret_req = NULL;
+
+	req = ldb_build_req_common(mem_ctx, ldb, controls,
+				   context, callback, parent);
+	if (req == NULL) {
+		ldb_set_errstring(ldb, "Out of Memory");
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	req->operation = LDB_ADD;
+	req->op.add.message = message;
+	*ret_req = req;
+	return LDB_SUCCESS;
+}
+
+int ldb_build_mod_req(struct ldb_request **ret_req,
+			struct ldb_context *ldb,
+			TALLOC_CTX *mem_ctx,
+			const struct ldb_message *message,
+			struct ldb_control **controls,
+			void *context,
+			ldb_request_callback_t callback,
+			struct ldb_request *parent)
+{
+	struct ldb_request *req;
+
+	*ret_req = NULL;
+
+	req = ldb_build_req_common(mem_ctx, ldb, controls,
+				   context, callback, parent);
+	if (req == NULL) {
+		ldb_set_errstring(ldb, "Out of Memory");
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	req->operation = LDB_MODIFY;
+	req->op.mod.message = message;
+
+	*ret_req = req;
+	return LDB_SUCCESS;
+}
+
+int ldb_build_del_req(struct ldb_request **ret_req,
+			struct ldb_context *ldb,
+			TALLOC_CTX *mem_ctx,
+			struct ldb_dn *dn,
+			struct ldb_control **controls,
+			void *context,
+			ldb_request_callback_t callback,
+			struct ldb_request *parent)
+{
+	struct ldb_request *req;
+
+	*ret_req = NULL;
+
+	req = ldb_build_req_common(mem_ctx, ldb, controls,
+				   context, callback, parent);
+	if (req == NULL) {
+		ldb_set_errstring(ldb, "Out of Memory");
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	req->operation = LDB_DELETE;
+	req->op.del.dn = dn;
+	*ret_req = req;
+	return LDB_SUCCESS;
+}
+
+int ldb_build_rename_req(struct ldb_request **ret_req,
+			struct ldb_context *ldb,
+			TALLOC_CTX *mem_ctx,
+			struct ldb_dn *olddn,
+			struct ldb_dn *newdn,
+			struct ldb_control **controls,
+			void *context,
+			ldb_request_callback_t callback,
+			struct ldb_request *parent)
+{
+	struct ldb_request *req;
+
+	*ret_req = NULL;
+
+	req = ldb_build_req_common(mem_ctx, ldb, controls,
+				   context, callback, parent);
+	if (req == NULL) {
+		ldb_set_errstring(ldb, "Out of Memory");
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	req->operation = LDB_RENAME;
+	req->op.rename.olddn = olddn;
+	req->op.rename.newdn = newdn;
+	*ret_req = req;
+	return LDB_SUCCESS;
+}
+
+int ldb_extended_default_callback(struct ldb_request *req,
+				  struct ldb_reply *ares)
+{
+	struct ldb_result *res;
+
+	res = talloc_get_type(req->context, struct ldb_result);
+
+	if (!ares) {
+		return ldb_request_done(req, LDB_ERR_OPERATIONS_ERROR);
+	}
+	if (ares->error != LDB_SUCCESS) {
+		return ldb_request_done(req, ares->error);
+	}
+
+	if (ares->type == LDB_REPLY_DONE) {
+
+		/* TODO: we should really support controls on entries and referrals too! */
+		res->extended = talloc_move(res, &ares->response);
+		res->controls = talloc_move(res, &ares->controls);
+
+		talloc_free(ares);
+		return ldb_request_done(req, LDB_SUCCESS);
+	}
+
+	talloc_free(ares);
+	ldb_asprintf_errstring(req->handle->ldb, "Invalid LDB reply type %d", ares->type);
+	return ldb_request_done(req, LDB_ERR_OPERATIONS_ERROR);
+}
+
+int ldb_build_extended_req(struct ldb_request **ret_req,
+			   struct ldb_context *ldb,
+			   TALLOC_CTX *mem_ctx,
+			   const char *oid,
+			   void *data,
+			   struct ldb_control **controls,
+			   void *context,
+			   ldb_request_callback_t callback,
+			   struct ldb_request *parent)
+{
+	struct ldb_request *req;
+
+	*ret_req = NULL;
+
+	req = ldb_build_req_common(mem_ctx, ldb, controls,
+				   context, callback, parent);
+	if (req == NULL) {
+		ldb_set_errstring(ldb, "Out of Memory");
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	req->operation = LDB_EXTENDED;
+	req->op.extended.oid = oid;
+	req->op.extended.data = data;
+	*ret_req = req;
+	return LDB_SUCCESS;
+}
+
+int ldb_extended(struct ldb_context *ldb,
+		 const char *oid,
+		 void *data,
+		 struct ldb_result **_res)
+{
+	struct ldb_request *req;
+	int ret;
+	struct ldb_result *res;
+
+	*_res = NULL;
+	req = NULL;
+
+	res = talloc_zero(ldb, struct ldb_result);
+	if (!res) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	ret = ldb_build_extended_req(&req, ldb, ldb,
+				     oid, data, NULL,
+				     res, ldb_extended_default_callback,
+				     NULL);
+	ldb_req_set_location(req, "ldb_extended");
+
+	if (ret != LDB_SUCCESS) goto done;
+
+	ldb_set_timeout(ldb, req, 0); /* use default timeout */
+
+	ret = ldb_request(ldb, req);
+
+	if (ret == LDB_SUCCESS) {
+		ret = ldb_wait(req->handle, LDB_WAIT_ALL);
+	}
+
+done:
+	if (ret != LDB_SUCCESS) {
+		talloc_free(res);
+		res = NULL;
+	}
+
+	talloc_free(req);
+
+	*_res = res;
+	return ret;
+}
+
+/*
+  note that ldb_search() will automatically replace a NULL 'base' value
+  with the defaultNamingContext from the rootDSE if available.
+*/
+int ldb_search(struct ldb_context *ldb, TALLOC_CTX *mem_ctx,
+		struct ldb_result **result, struct ldb_dn *base,
+		enum ldb_scope scope, const char * const *attrs,
+		const char *exp_fmt, ...)
+{
+	struct ldb_request *req;
+	struct ldb_result *res;
+	char *expression;
+	va_list ap;
+	int ret;
+
+	expression = NULL;
+	*result = NULL;
+	req = NULL;
+
+	res = talloc_zero(mem_ctx, struct ldb_result);
+	if (!res) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	if (exp_fmt) {
+		va_start(ap, exp_fmt);
+		expression = talloc_vasprintf(mem_ctx, exp_fmt, ap);
+		va_end(ap);
+
+		if (!expression) {
+			talloc_free(res);
+			return LDB_ERR_OPERATIONS_ERROR;
+		}
+	}
+
+	ret = ldb_build_search_req(&req, ldb, mem_ctx,
+					base?base:ldb_get_default_basedn(ldb),
+	       				scope,
+					expression,
+					attrs,
+					NULL,
+					res,
+					ldb_search_default_callback,
+					NULL);
+	ldb_req_set_location(req, "ldb_search");
+
+	if (ret != LDB_SUCCESS) goto done;
+
+	ret = ldb_request(ldb, req);
+
+	if (ret == LDB_SUCCESS) {
+		ret = ldb_wait(req->handle, LDB_WAIT_ALL);
+	}
+
+done:
+	if (ret != LDB_SUCCESS) {
+		talloc_free(res);
+		res = NULL;
+	}
+
+	talloc_free(expression);
+	talloc_free(req);
+
+	*result = res;
+	return ret;
+}
+
+/*
+  add a record to the database. Will fail if a record with the given class
+  and key already exists
+*/
+int ldb_add(struct ldb_context *ldb,
+	    const struct ldb_message *message)
+{
+	struct ldb_request *req;
+	int ret;
+
+	ret = ldb_msg_sanity_check(ldb, message);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	ret = ldb_build_add_req(&req, ldb, ldb,
+					message,
+					NULL,
+					NULL,
+					ldb_op_default_callback,
+					NULL);
+	ldb_req_set_location(req, "ldb_add");
+
+	if (ret != LDB_SUCCESS) return ret;
+
+	/* do request and autostart a transaction */
+	ret = ldb_autotransaction_request(ldb, req);
+
+	talloc_free(req);
+	return ret;
+}
+
+/*
+  modify the specified attributes of a record
+*/
+int ldb_modify(struct ldb_context *ldb,
+	       const struct ldb_message *message)
+{
+	struct ldb_request *req;
+	int ret;
+
+	ret = ldb_msg_sanity_check(ldb, message);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	ret = ldb_build_mod_req(&req, ldb, ldb,
+					message,
+					NULL,
+					NULL,
+					ldb_op_default_callback,
+					NULL);
+	ldb_req_set_location(req, "ldb_modify");
+
+	if (ret != LDB_SUCCESS) return ret;
+
+	/* do request and autostart a transaction */
+	ret = ldb_autotransaction_request(ldb, req);
+
+	talloc_free(req);
+	return ret;
+}
+
+
+/*
+  delete a record from the database
+*/
+int ldb_delete(struct ldb_context *ldb, struct ldb_dn *dn)
+{
+	struct ldb_request *req;
+	int ret;
+
+	ret = ldb_build_del_req(&req, ldb, ldb,
+					dn,
+					NULL,
+					NULL,
+					ldb_op_default_callback,
+					NULL);
+	ldb_req_set_location(req, "ldb_delete");
+
+	if (ret != LDB_SUCCESS) return ret;
+
+	/* do request and autostart a transaction */
+	ret = ldb_autotransaction_request(ldb, req);
+
+	talloc_free(req);
+	return ret;
+}
+
+/*
+  rename a record in the database
+*/
+int ldb_rename(struct ldb_context *ldb,
+		struct ldb_dn *olddn, struct ldb_dn *newdn)
+{
+	struct ldb_request *req;
+	int ret;
+
+	ret = ldb_build_rename_req(&req, ldb, ldb,
+					olddn,
+					newdn,
+					NULL,
+					NULL,
+					ldb_op_default_callback,
+					NULL);
+	ldb_req_set_location(req, "ldb_rename");
+
+	if (ret != LDB_SUCCESS) return ret;
+
+	/* do request and autostart a transaction */
+	ret = ldb_autotransaction_request(ldb, req);
+
+	talloc_free(req);
+	return ret;
+}
+
+
+/*
+  return the global sequence number
+*/
+int ldb_sequence_number(struct ldb_context *ldb,
+			enum ldb_sequence_type type, uint64_t *seq_num)
+{
+	struct ldb_seqnum_request *seq;
+	struct ldb_seqnum_result *seqr;
+	struct ldb_result *res;
+	TALLOC_CTX *tmp_ctx;
+	int ret;
+
+	*seq_num = 0;
+
+	tmp_ctx = talloc_zero(ldb, struct ldb_request);
+	if (tmp_ctx == NULL) {
+		ldb_set_errstring(ldb, "Out of Memory");
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	seq = talloc_zero(tmp_ctx, struct ldb_seqnum_request);
+	if (seq == NULL) {
+		ldb_set_errstring(ldb, "Out of Memory");
+		ret = LDB_ERR_OPERATIONS_ERROR;
+		goto done;
+	}
+	seq->type = type;
+
+	ret = ldb_extended(ldb, LDB_EXTENDED_SEQUENCE_NUMBER, seq, &res);
+	if (ret != LDB_SUCCESS) {
+		goto done;
+	}
+	talloc_steal(tmp_ctx, res);
+
+	if (strcmp(LDB_EXTENDED_SEQUENCE_NUMBER, res->extended->oid) != 0) {
+		ldb_set_errstring(ldb, "Invalid OID in reply");
+		ret = LDB_ERR_OPERATIONS_ERROR;
+		goto done;
+	}
+	seqr = talloc_get_type(res->extended->data,
+				struct ldb_seqnum_result);
+	*seq_num = seqr->seq_num;
+
+done:
+	talloc_free(tmp_ctx);
+	return ret;
+}
+
+/*
+  return extended error information
+*/
+const char *ldb_errstring(struct ldb_context *ldb)
+{
+	if (ldb->err_string) {
+		return ldb->err_string;
+	}
+
+	return NULL;
+}
+
+/*
+  return a string explaining what a ldb error constant meancs
+*/
+const char *ldb_strerror(int ldb_err)
+{
+	switch (ldb_err) {
+	case LDB_SUCCESS:
+		return "Success";
+	case LDB_ERR_OPERATIONS_ERROR:
+		return "Operations error";
+	case LDB_ERR_PROTOCOL_ERROR:
+		return "Protocol error";
+	case LDB_ERR_TIME_LIMIT_EXCEEDED:
+		return "Time limit exceeded";
+	case LDB_ERR_SIZE_LIMIT_EXCEEDED:
+		return "Size limit exceeded";
+	case LDB_ERR_COMPARE_FALSE:
+		return "Compare false";
+	case LDB_ERR_COMPARE_TRUE:
+		return "Compare true";
+	case LDB_ERR_AUTH_METHOD_NOT_SUPPORTED:
+		return "Auth method not supported";
+	case LDB_ERR_STRONG_AUTH_REQUIRED:
+		return "Strong auth required";
+/* 9 RESERVED */
+	case LDB_ERR_REFERRAL:
+		return "Referral error";
+	case LDB_ERR_ADMIN_LIMIT_EXCEEDED:
+		return "Admin limit exceeded";
+	case LDB_ERR_UNSUPPORTED_CRITICAL_EXTENSION:
+		return "Unsupported critical extension";
+	case LDB_ERR_CONFIDENTIALITY_REQUIRED:
+		return "Confidentiality required";
+	case LDB_ERR_SASL_BIND_IN_PROGRESS:
+		return "SASL bind in progress";
+	case LDB_ERR_NO_SUCH_ATTRIBUTE:
+		return "No such attribute";
+	case LDB_ERR_UNDEFINED_ATTRIBUTE_TYPE:
+		return "Undefined attribute type";
+	case LDB_ERR_INAPPROPRIATE_MATCHING:
+		return "Inappropriate matching";
+	case LDB_ERR_CONSTRAINT_VIOLATION:
+		return "Constraint violation";
+	case LDB_ERR_ATTRIBUTE_OR_VALUE_EXISTS:
+		return "Attribute or value exists";
+	case LDB_ERR_INVALID_ATTRIBUTE_SYNTAX:
+		return "Invalid attribute syntax";
+/* 22-31 unused */
+	case LDB_ERR_NO_SUCH_OBJECT:
+		return "No such object";
+	case LDB_ERR_ALIAS_PROBLEM:
+		return "Alias problem";
+	case LDB_ERR_INVALID_DN_SYNTAX:
+		return "Invalid DN syntax";
+/* 35 RESERVED */
+	case LDB_ERR_ALIAS_DEREFERENCING_PROBLEM:
+		return "Alias dereferencing problem";
+/* 37-47 unused */
+	case LDB_ERR_INAPPROPRIATE_AUTHENTICATION:
+		return "Inappropriate authentication";
+	case LDB_ERR_INVALID_CREDENTIALS:
+		return "Invalid credentials";
+	case LDB_ERR_INSUFFICIENT_ACCESS_RIGHTS:
+		return "insufficient access rights";
+	case LDB_ERR_BUSY:
+		return "Busy";
+	case LDB_ERR_UNAVAILABLE:
+		return "Unavailable";
+	case LDB_ERR_UNWILLING_TO_PERFORM:
+		return "Unwilling to perform";
+	case LDB_ERR_LOOP_DETECT:
+		return "Loop detect";
+/* 55-63 unused */
+	case LDB_ERR_NAMING_VIOLATION:
+		return "Naming violation";
+	case LDB_ERR_OBJECT_CLASS_VIOLATION:
+		return "Object class violation";
+	case LDB_ERR_NOT_ALLOWED_ON_NON_LEAF:
+		return "Not allowed on non-leaf";
+	case LDB_ERR_NOT_ALLOWED_ON_RDN:
+		return "Not allowed on RDN";
+	case LDB_ERR_ENTRY_ALREADY_EXISTS:
+		return "Entry already exists";
+	case LDB_ERR_OBJECT_CLASS_MODS_PROHIBITED:
+		return "Object class mods prohibited";
+/* 70 RESERVED FOR CLDAP */
+	case LDB_ERR_AFFECTS_MULTIPLE_DSAS:
+		return "Affects multiple DSAs";
+/* 72-79 unused */
+	case LDB_ERR_OTHER:
+		return "Other";
+	}
+
+	return "Unknown error";
+}
+
+/*
+  set backend specific opaque parameters
+*/
+int ldb_set_opaque(struct ldb_context *ldb, const char *name, void *value)
+{
+	struct ldb_opaque *o;
+
+	/* allow updating an existing value */
+	for (o=ldb->opaque;o;o=o->next) {
+		if (strcmp(o->name, name) == 0) {
+			o->value = value;
+			return LDB_SUCCESS;
+		}
+	}
+
+	o = talloc(ldb, struct ldb_opaque);
+	if (o == NULL) {
+		ldb_oom(ldb);
+		return LDB_ERR_OTHER;
+	}
+	o->next = ldb->opaque;
+	o->name = name;
+	o->value = value;
+	ldb->opaque = o;
+	return LDB_SUCCESS;
+}
+
+/*
+  get a previously set opaque value
+*/
+void *ldb_get_opaque(struct ldb_context *ldb, const char *name)
+{
+	struct ldb_opaque *o;
+	for (o=ldb->opaque;o;o=o->next) {
+		if (strcmp(o->name, name) == 0) {
+			return o->value;
+		}
+	}
+	return NULL;
+}
+
+int ldb_global_init(void)
+{
+	/* Provided for compatibility with some older versions of ldb */
+	return 0;
+}
+
+/* return the ldb flags */
+unsigned int ldb_get_flags(struct ldb_context *ldb)
+{
+	return ldb->flags;
+}
+
+/* set the ldb flags */
+void ldb_set_flags(struct ldb_context *ldb, unsigned flags)
+{
+	ldb->flags = flags;
+}
+
+
+/*
+  set the location in a ldb request. Used for debugging
+ */
+void ldb_req_set_location(struct ldb_request *req, const char *location)
+{
+	if (req && req->handle) {
+		req->handle->location = location;
+	}
+}
+
+/*
+  return the location set with dsdb_req_set_location
+ */
+const char *ldb_req_location(struct ldb_request *req)
+{
+	return req->handle->location;
+}
+
+/**
+  mark a request as untrusted. This tells the rootdse module to remove
+  unregistered controls
+ */
+void ldb_req_mark_untrusted(struct ldb_request *req)
+{
+	req->handle->flags |= LDB_HANDLE_FLAG_UNTRUSTED;
+}
+
+/**
+  mark a request as trusted.
+ */
+void ldb_req_mark_trusted(struct ldb_request *req)
+{
+	req->handle->flags &= ~LDB_HANDLE_FLAG_UNTRUSTED;
+}
+
+/**
+  set custom flags. Those flags are set by applications using ldb,
+  they are application dependent and the same bit can have different
+  meaning in different application.
+ */
+void ldb_req_set_custom_flags(struct ldb_request *req, uint32_t flags)
+{
+	if (req != NULL && req->handle != NULL) {
+		req->handle->custom_flags = flags;
+	}
+}
+
+
+/**
+  get custom flags. Those flags are set by applications using ldb,
+  they are application dependent and the same bit can have different
+  meaning in different application.
+ */
+uint32_t ldb_req_get_custom_flags(struct ldb_request *req)
+{
+	if (req != NULL && req->handle != NULL) {
+		return req->handle->custom_flags;
+	}
+
+	/*
+	 * 0 is not something any better or worse than
+	 * anything else as req or the handle is NULL
+	 */
+	return 0;
+}
+
+
+/**
+ * return true if a request is untrusted
+ */
+bool ldb_req_is_untrusted(struct ldb_request *req)
+{
+	return (req->handle->flags & LDB_HANDLE_FLAG_UNTRUSTED) != 0;
+}
diff --git a/lib/ldb/common/ldb_attributes.c b/lib/ldb/common/ldb_attributes.c
new file mode 100644
index 0000000..32f25fd
--- /dev/null
+++ b/lib/ldb/common/ldb_attributes.c
@@ -0,0 +1,411 @@
+/* 
+   ldb database library
+
+   Copyright (C) Andrew Tridgell  2005
+
+     ** NOTE! The following LGPL license applies to the ldb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+   
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+/*
+  register handlers for specific attributes and objectclass relationships
+
+  this allows a backend to store its schema information in any format
+  it likes (or to not have any schema information at all) while keeping the 
+  message matching logic generic
+*/
+
+#include "ldb_private.h"
+#include "ldb_handlers.h"
+
+/*
+  fill in an attribute to the ldb_schema into the supplied buffer
+
+  if flags contains LDB_ATTR_FLAG_ALLOCATED
+  the attribute name string will be copied using
+  talloc_strdup(), otherwise it needs to be a static const
+  string at least with a lifetime longer than the ldb struct!
+
+  the ldb_schema_syntax structure should be a pointer
+  to a static const struct or at least it needs to be
+  a struct with a longer lifetime than the ldb context!
+
+*/
+int ldb_schema_attribute_fill_with_syntax(struct ldb_context *ldb,
+					  TALLOC_CTX *mem_ctx,
+					  const char *attribute,
+					  unsigned flags,
+					  const struct ldb_schema_syntax *syntax,
+					  struct ldb_schema_attribute *a)
+{
+	a->name	= attribute;
+	a->flags	= flags;
+	a->syntax	= syntax;
+
+	if (a->flags & LDB_ATTR_FLAG_ALLOCATED) {
+		a->name = talloc_strdup(mem_ctx, a->name);
+		if (a->name == NULL) {
+			ldb_oom(ldb);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+/*
+  add a attribute to the ldb_schema
+
+  if flags contains LDB_ATTR_FLAG_ALLOCATED
+  the attribute name string will be copied using
+  talloc_strdup(), otherwise it needs to be a static const
+  string at least with a lifetime longer than the ldb struct!
+  
+  the ldb_schema_syntax structure should be a pointer
+  to a static const struct or at least it needs to be
+  a struct with a longer lifetime than the ldb context!
+
+*/
+int ldb_schema_attribute_add_with_syntax(struct ldb_context *ldb, 
+					 const char *attribute,
+					 unsigned flags,
+					 const struct ldb_schema_syntax *syntax)
+{
+	unsigned int i, n;
+	struct ldb_schema_attribute *a;
+
+	if (!syntax) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	n = ldb->schema.num_attributes + 1;
+
+	a = talloc_realloc(ldb, ldb->schema.attributes,
+			   struct ldb_schema_attribute, n);
+	if (a == NULL) {
+		ldb_oom(ldb);
+		return -1;
+	}
+	ldb->schema.attributes = a;
+
+	for (i = 0; i < ldb->schema.num_attributes; i++) {
+		int cmp = ldb_attr_cmp(attribute, a[i].name);
+		if (cmp == 0) {
+			/* silently ignore attempts to overwrite fixed attributes */
+			if (a[i].flags & LDB_ATTR_FLAG_FIXED) {
+				return 0;
+			}
+			if (a[i].flags & LDB_ATTR_FLAG_ALLOCATED) {
+				talloc_free(discard_const_p(char, a[i].name));
+			}
+			/* To cancel out increment below */
+			ldb->schema.num_attributes--;
+			break;
+		} else if (cmp < 0) {
+			memmove(a+i+1, a+i, sizeof(*a) * (ldb->schema.num_attributes-i));
+			break;
+		}
+	}
+	ldb->schema.num_attributes++;
+
+	a[i].name	= attribute;
+	a[i].flags	= flags;
+	a[i].syntax	= syntax;
+
+	if (a[i].flags & LDB_ATTR_FLAG_ALLOCATED) {
+		a[i].name = talloc_strdup(a, a[i].name);
+		if (a[i].name == NULL) {
+			ldb_oom(ldb);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+static const struct ldb_schema_syntax ldb_syntax_default = {
+	.name            = LDB_SYNTAX_OCTET_STRING,
+	.ldif_read_fn    = ldb_handler_copy,
+	.ldif_write_fn   = ldb_handler_copy,
+	.canonicalise_fn = ldb_handler_copy,
+	.comparison_fn   = ldb_comparison_binary
+};
+
+static const struct ldb_schema_attribute ldb_attribute_default = {
+	.name	= NULL,
+	.flags	= 0,
+	.syntax	= &ldb_syntax_default
+};
+
+/*
+ * Return the attribute handlers for a given attribute
+ *
+ * @param ldb	ldb context
+ * @param name	attribute name to search for
+ * @return	Always return valid pointer to schema attribute.
+ * 		In case there is no attribute with name,
+ * 		ldb_attribute_default is returned
+ */
+static const struct ldb_schema_attribute *ldb_schema_attribute_by_name_internal(
+	struct ldb_context *ldb,
+	const char *name)
+{
+	/* for binary search we need signed variables */
+	unsigned int i, e, b = 0;
+	int r;
+	const struct ldb_schema_attribute *def = &ldb_attribute_default;
+
+	/* fallback to default attribute implementation */
+	if (name == NULL) {
+		return def;
+	}
+
+	/* as handlers are sorted, '*' must be the first if present */
+	if (strcmp(ldb->schema.attributes[0].name, "*") == 0) {
+		def = &ldb->schema.attributes[0];
+		b = 1;
+	}
+
+	/* do a binary search on the array */
+	e = ldb->schema.num_attributes - 1;
+
+	while ((b <= e) && (e != (unsigned int) -1)) {
+		i = (b + e) / 2;
+
+		r = ldb_attr_cmp(name, ldb->schema.attributes[i].name);
+		if (r == 0) {
+			return &ldb->schema.attributes[i];
+		}
+		if (r < 0) {
+			e = i - 1;
+		} else {
+			b = i + 1;
+		}
+	}
+
+	return def;
+}
+
+/*
+  return the attribute handlers for a given attribute
+*/
+const struct ldb_schema_attribute *ldb_schema_attribute_by_name(struct ldb_context *ldb,
+								const char *name)
+{
+	if (ldb->schema.attribute_handler_override) {
+		const struct ldb_schema_attribute *ret = 
+			ldb->schema.attribute_handler_override(ldb, 
+							       ldb->schema.attribute_handler_override_private,
+							       name);
+		if (ret) {
+			return ret;
+		}
+	}
+
+	return ldb_schema_attribute_by_name_internal(ldb, name);
+}
+
+
+/*
+  add to the list of ldif handlers for this ldb context
+*/
+void ldb_schema_attribute_remove(struct ldb_context *ldb, const char *name)
+{
+	const struct ldb_schema_attribute *a;
+	ptrdiff_t i;
+
+	a = ldb_schema_attribute_by_name_internal(ldb, name);
+	if (a == NULL || a->name == NULL) {
+		return;
+	}
+
+	/* FIXED attributes are never removed */
+	if (a->flags & LDB_ATTR_FLAG_FIXED) {
+		return;
+	}
+
+	if (a->flags & LDB_ATTR_FLAG_ALLOCATED) {
+		talloc_free(discard_const_p(char, a->name));
+	}
+
+	i = a - ldb->schema.attributes;
+	if (i < ldb->schema.num_attributes - 1) {
+		memmove(&ldb->schema.attributes[i], 
+			a+1, sizeof(*a) * (ldb->schema.num_attributes-(i+1)));
+	}
+
+	ldb->schema.num_attributes--;
+}
+
+/*
+  remove attributes with a specified flag (eg LDB_ATTR_FLAG_FROM_DB) for this ldb context
+
+  This is to permit correct reloads
+*/
+void ldb_schema_attribute_remove_flagged(struct ldb_context *ldb, unsigned int flag)
+{
+	ptrdiff_t i;
+
+	for (i = 0; i < ldb->schema.num_attributes;) {
+		const struct ldb_schema_attribute *a
+			= &ldb->schema.attributes[i];
+		/* FIXED attributes are never removed */
+		if (a->flags & LDB_ATTR_FLAG_FIXED) {
+			i++;
+			continue;
+		}
+		if ((a->flags & flag) == 0) {
+			i++;
+			continue;
+		}
+		if (a->flags & LDB_ATTR_FLAG_ALLOCATED) {
+			talloc_free(discard_const_p(char, a->name));
+		}
+		if (i < ldb->schema.num_attributes - 1) {
+			memmove(&ldb->schema.attributes[i],
+				a+1, sizeof(*a) * (ldb->schema.num_attributes-(i+1)));
+		}
+
+		ldb->schema.num_attributes--;
+	}
+}
+
+/*
+  setup a attribute handler using a standard syntax
+*/
+int ldb_schema_attribute_add(struct ldb_context *ldb,
+			     const char *attribute,
+			     unsigned flags,
+			     const char *syntax)
+{
+	const struct ldb_schema_syntax *s = ldb_standard_syntax_by_name(ldb, syntax);
+	return ldb_schema_attribute_add_with_syntax(ldb, attribute, flags, s);
+}
+
+/*
+  setup the attribute handles for well known attributes
+*/
+int ldb_setup_wellknown_attributes(struct ldb_context *ldb)
+{
+	const struct {
+		const char *attr;
+		const char *syntax;
+	} wellknown[] = {
+		{ "dn", LDB_SYNTAX_DN },
+		{ "distinguishedName", LDB_SYNTAX_DN },
+		{ "cn", LDB_SYNTAX_DIRECTORY_STRING },
+		{ "dc", LDB_SYNTAX_DIRECTORY_STRING },
+		{ "ou", LDB_SYNTAX_DIRECTORY_STRING },
+		{ "objectClass", LDB_SYNTAX_OBJECTCLASS }
+	};
+	unsigned int i;
+	int ret;
+
+	for (i=0;i<ARRAY_SIZE(wellknown);i++) {
+		ret = ldb_schema_attribute_add(ldb, wellknown[i].attr, 0,
+					       wellknown[i].syntax);
+		if (ret != LDB_SUCCESS) {
+			return ret;
+		}
+	}
+
+	return LDB_SUCCESS;
+}
+
+
+/*
+  add a extended dn syntax to the ldb_schema
+*/
+int ldb_dn_extended_add_syntax(struct ldb_context *ldb, 
+			       unsigned flags,
+			       const struct ldb_dn_extended_syntax *syntax)
+{
+	unsigned int n;
+	struct ldb_dn_extended_syntax *a;
+
+	if (!syntax) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	n = ldb->schema.num_dn_extended_syntax + 1;
+
+	a = talloc_realloc(ldb, ldb->schema.dn_extended_syntax,
+			   struct ldb_dn_extended_syntax, n);
+
+	if (!a) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	a[ldb->schema.num_dn_extended_syntax] = *syntax;
+	ldb->schema.dn_extended_syntax = a;
+
+	ldb->schema.num_dn_extended_syntax = n;
+
+	return LDB_SUCCESS;
+}
+
+/*
+  return the extended dn syntax for a given name
+*/
+const struct ldb_dn_extended_syntax *ldb_dn_extended_syntax_by_name(struct ldb_context *ldb,
+								    const char *name)
+{
+	unsigned int i;
+	for (i=0; i < ldb->schema.num_dn_extended_syntax; i++) {
+		if (ldb_attr_cmp(ldb->schema.dn_extended_syntax[i].name, name) == 0) {
+			return &ldb->schema.dn_extended_syntax[i];
+		}
+	}
+	return NULL;
+}
+
+/*
+  set an attribute handler override function - used to delegate schema handling
+  to external code
+ */
+void ldb_schema_attribute_set_override_handler(struct ldb_context *ldb,
+					       ldb_attribute_handler_override_fn_t override,
+					       void *private_data)
+{
+	ldb->schema.attribute_handler_override_private = private_data;
+	ldb->schema.attribute_handler_override = override;
+}
+
+/*
+  set that the attribute handler override function - used to delegate
+  schema handling to external code, is handling setting
+  LDB_ATTR_FLAG_INDEXED
+ */
+void ldb_schema_set_override_indexlist(struct ldb_context *ldb,
+				       bool one_level_indexes)
+{
+	ldb->schema.index_handler_override = true;
+	ldb->schema.one_level_indexes = one_level_indexes;
+}
+
+/*
+ * set that the GUID index mode is in operation
+ *
+ * The caller must ensure the supplied strings do not go out of
+ * scope (they are typically constant memory).
+ */
+void ldb_schema_set_override_GUID_index(struct ldb_context *ldb,
+					const char *GUID_index_attribute,
+					const char *GUID_index_dn_component)
+{
+	ldb->schema.GUID_index_attribute = GUID_index_attribute;
+	ldb->schema.GUID_index_dn_component = GUID_index_dn_component;
+}
diff --git a/lib/ldb/common/ldb_controls.c b/lib/ldb/common/ldb_controls.c
new file mode 100644
index 0000000..266aa90
--- /dev/null
+++ b/lib/ldb/common/ldb_controls.c
@@ -0,0 +1,1342 @@
+/* 
+   ldb database library
+
+   Copyright (C) Simo Sorce  2005
+
+     ** NOTE! The following LGPL license applies to the ldb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+   
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+/*
+ *  Name: ldb_controls.c
+ *
+ *  Component: ldb controls utility functions
+ *
+ *  Description: helper functions for control modules
+ *
+ *  Author: Simo Sorce
+ */
+
+#include "ldb_private.h"
+
+/* check if a control with the specified "oid" exist and return it */
+/* returns NULL if not found */
+struct ldb_control *ldb_request_get_control(struct ldb_request *req, const char *oid)
+{
+	unsigned int i;
+
+	if (req->controls != NULL) {
+		for (i = 0; req->controls[i]; i++) {
+			if (req->controls[i]->oid && strcmp(oid, req->controls[i]->oid) == 0) {
+				break;
+			}
+		}
+
+		return req->controls[i];
+	}
+
+	return NULL;
+}
+
+/* check if a control with the specified "oid" exist and return it */
+/* returns NULL if not found */
+struct ldb_control *ldb_reply_get_control(struct ldb_reply *rep, const char *oid)
+{
+	unsigned int i;
+
+	if (rep->controls != NULL) {
+		for (i = 0; rep->controls[i]; i++) {
+			if (rep->controls[i]->oid && strcmp(oid, rep->controls[i]->oid) == 0) {
+				break;
+			}
+		}
+
+		return rep->controls[i];
+	}
+
+	return NULL;
+}
+
+/*
+ * Saves the current controls list into the "saver" (can also be NULL) and
+ * replace the one in "req" with a new one excluding the "exclude" control
+ * (if it is NULL then the list remains the same)
+ *
+ * Returns 0 on error.
+ */
+int ldb_save_controls(struct ldb_control *exclude, struct ldb_request *req, struct ldb_control ***saver)
+{
+	struct ldb_control **lcs, **lcs_old;
+	unsigned int i, j;
+
+	lcs_old = req->controls;
+	if (saver != NULL) {
+		*saver = lcs_old;
+	}
+
+	for (i = 0; req->controls && req->controls[i]; i++);
+	if (i == 0) {
+		req->controls = NULL;
+		return 1;
+	}
+
+	lcs = talloc_array(req, struct ldb_control *, i + 1);
+	if (!lcs) {
+		return 0;
+	}
+
+	for (i = 0, j = 0; lcs_old[i]; i++) {
+		if (exclude == lcs_old[i]) continue;
+		lcs[j] = lcs_old[i];
+		j++;
+	}
+	lcs[j] = NULL;
+
+	req->controls = talloc_realloc(req, lcs, struct ldb_control *, j + 1);
+	if (req->controls == NULL) {
+		return 0;
+	}
+	return 1;
+}
+
+/*
+ * Returns a list of controls, except the one specified with "exclude" (can
+ * also be NULL).  Included controls become a child of returned list if they
+ * were children of "controls_in".
+ *
+ * Returns NULL on error (OOM) or an empty control list.
+ */
+struct ldb_control **ldb_controls_except_specified(struct ldb_control **controls_in, 
+					       TALLOC_CTX *mem_ctx, 
+					       struct ldb_control *exclude)
+{
+	struct ldb_control **lcs = NULL;
+	unsigned int i, j, n;
+
+	for (i = 0; controls_in && controls_in[i]; i++);
+	if (i == 0) {
+		return NULL;
+	}
+	n = i;
+
+	for (i = 0, j = 0; controls_in && controls_in[i]; i++) {
+		if (exclude == controls_in[i]) continue;
+
+		if (!lcs) {
+			/* Allocate here so if we remove the only
+			 * control, or there were no controls, we
+			 * don't allocate at all, and just return
+			 * NULL */
+			lcs = talloc_array(mem_ctx, struct ldb_control *,
+					   n + 1);
+			if (!lcs) {
+				return NULL;
+			}
+		}
+
+		lcs[j] = controls_in[i];
+		talloc_reparent(controls_in, lcs, lcs[j]);
+		j++;
+	}
+	if (lcs) {
+		lcs[j] = NULL;
+
+		lcs = talloc_realloc(mem_ctx, lcs, struct ldb_control *, j + 1);
+	}
+
+	return lcs;
+}
+
+/* check if there's any control marked as critical in the list */
+/* return True if any, False if none */
+int ldb_check_critical_controls(struct ldb_control **controls)
+{
+	unsigned int i;
+
+	if (controls == NULL) {
+		return 0;
+	}
+
+	for (i = 0; controls[i]; i++) {
+		if (controls[i]->critical) {
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+int ldb_request_add_control(struct ldb_request *req, const char *oid, bool critical, void *data)
+{
+	unsigned int i, n;
+	struct ldb_control **ctrls;
+	struct ldb_control *ctrl;
+
+	for (n=0; req->controls && req->controls[n];n++) { 
+		/* having two controls of the same OID makes no sense */
+		if (req->controls[n]->oid && strcmp(oid, req->controls[n]->oid) == 0) {
+			return LDB_ERR_ATTRIBUTE_OR_VALUE_EXISTS;
+		}
+	}
+
+	ctrls = talloc_array(req,
+			       struct ldb_control *,
+			       n + 2);
+	if (!ctrls) return LDB_ERR_OPERATIONS_ERROR;
+
+	for (i=0; i<n; i++) {
+		ctrls[i] = req->controls[i];
+	}
+
+	req->controls = ctrls;
+	ctrls[n] = NULL;
+	ctrls[n+1] = NULL;
+
+	ctrl = talloc(ctrls, struct ldb_control);
+	if (!ctrl) return LDB_ERR_OPERATIONS_ERROR;
+
+	ctrl->oid	= talloc_strdup(ctrl, oid);
+	if (!ctrl->oid) return LDB_ERR_OPERATIONS_ERROR;
+	ctrl->critical	= critical;
+	ctrl->data	= data;
+
+	ctrls[n] = ctrl;
+	return LDB_SUCCESS;
+}
+
+int ldb_reply_add_control(struct ldb_reply *ares, const char *oid, bool critical, void *data)
+{
+	unsigned n;
+	struct ldb_control **ctrls;
+	struct ldb_control *ctrl;
+
+	for (n=0; ares->controls && ares->controls[n];) { 
+		/* having two controls of the same OID makes no sense */
+		if (ares->controls[n]->oid && strcmp(oid, ares->controls[n]->oid) == 0) {
+			return LDB_ERR_ATTRIBUTE_OR_VALUE_EXISTS;
+		}
+		n++; 
+	}
+
+	ctrls = talloc_realloc(ares, ares->controls,
+			       struct ldb_control *,
+			       n + 2);
+	if (!ctrls) return LDB_ERR_OPERATIONS_ERROR;
+	ares->controls = ctrls;
+	ctrls[n] = NULL;
+	ctrls[n+1] = NULL;
+
+	ctrl = talloc(ctrls, struct ldb_control);
+	if (!ctrl) return LDB_ERR_OPERATIONS_ERROR;
+
+	ctrl->oid	= talloc_strdup(ctrl, oid);
+	if (!ctrl->oid) return LDB_ERR_OPERATIONS_ERROR;
+	ctrl->critical	= critical;
+	ctrl->data	= data;
+
+	ctrls[n] = ctrl;
+	return LDB_SUCCESS;
+}
+
+/* Add a control to the request, replacing the old one if it is already in the request */
+int ldb_request_replace_control(struct ldb_request *req, const char *oid, bool critical, void *data)
+{
+	unsigned int n;
+	int ret;
+
+	ret = ldb_request_add_control(req, oid, critical, data);
+	if (ret != LDB_ERR_ATTRIBUTE_OR_VALUE_EXISTS) {
+		return ret;
+	}
+
+	for (n=0; req->controls[n];n++) {
+		if (req->controls[n]->oid && strcmp(oid, req->controls[n]->oid) == 0) {
+			req->controls[n]->critical = critical;
+			req->controls[n]->data = data;
+			return LDB_SUCCESS;
+		}
+	}
+
+	return LDB_ERR_OPERATIONS_ERROR;
+}
+
+/*
+ * Return a control as string
+ * the project (ie. name:value1:value2:...:valuen
+ * The string didn't include the criticity of the critical flag
+ */
+char *ldb_control_to_string(TALLOC_CTX *mem_ctx, const struct ldb_control *control)
+{
+	char *res = NULL;
+
+	if (strcmp(control->oid, LDB_CONTROL_PAGED_RESULTS_OID) == 0) {
+		struct ldb_paged_control *rep_control = talloc_get_type(control->data, struct ldb_paged_control);
+		char *cookie;
+		if (rep_control == NULL) {
+			return NULL;
+		}
+
+		cookie = ldb_base64_encode(mem_ctx, rep_control->cookie, rep_control->cookie_len);
+		if (cookie == NULL) {
+			return NULL;
+		}
+		if (cookie[0] != '\0') {
+			res = talloc_asprintf(mem_ctx, "%s:%d:%s",
+						LDB_CONTROL_PAGED_RESULTS_NAME,
+						control->critical,
+						cookie);
+
+			talloc_free(cookie);
+		} else {
+			res = talloc_asprintf(mem_ctx, "%s:%d",
+						LDB_CONTROL_PAGED_RESULTS_NAME,
+						control->critical);
+		}
+		return res;
+	}
+
+	if (strcmp(control->oid, LDB_CONTROL_VLV_RESP_OID) == 0) {
+		struct ldb_vlv_resp_control *rep_control = talloc_get_type(control->data,
+								struct ldb_vlv_resp_control);
+
+		char *cookie;
+
+		if (rep_control == NULL) {
+			return NULL;
+		}
+
+		cookie = ldb_base64_encode(mem_ctx,
+					   (char *)rep_control->contextId,
+					   rep_control->ctxid_len);
+		if (cookie == NULL) {
+			return NULL;
+		}
+
+		res = talloc_asprintf(mem_ctx, "%s:%d:%d:%d:%d:%s",
+						LDB_CONTROL_VLV_RESP_NAME,
+						control->critical,
+						rep_control->targetPosition,
+						rep_control->contentCount,
+						rep_control->vlv_result,
+				                cookie);
+
+		return res;
+	}
+
+	if (strcmp(control->oid, LDB_CONTROL_SORT_RESP_OID) == 0) {
+		struct ldb_sort_resp_control *rep_control = talloc_get_type(control->data,
+								struct ldb_sort_resp_control);
+
+		if (rep_control == NULL) {
+			return NULL;
+		}
+		res = talloc_asprintf(mem_ctx, "%s:%d:%d:%s",
+					LDB_CONTROL_SORT_RESP_NAME,
+					control->critical,
+					rep_control->result,
+					rep_control->attr_desc);
+
+		return res;
+	}
+
+	if (strcmp(control->oid, LDB_CONTROL_ASQ_OID) == 0) {
+		struct ldb_asq_control *rep_control = talloc_get_type(control->data,
+								struct ldb_asq_control);
+
+		if (rep_control == NULL) {
+			return NULL;
+		}
+		res = talloc_asprintf(mem_ctx, "%s:%d:%d",
+					LDB_CONTROL_SORT_RESP_NAME,
+					control->critical,
+					rep_control->result);
+
+		return res;
+	}
+
+	if (strcmp(control->oid, LDB_CONTROL_DIRSYNC_OID) == 0) {
+		char *cookie;
+		struct ldb_dirsync_control *rep_control = talloc_get_type(control->data,
+								struct ldb_dirsync_control);
+
+		if (rep_control == NULL) {
+			return NULL;
+		}
+		cookie = ldb_base64_encode(mem_ctx, rep_control->cookie,
+				rep_control->cookie_len);
+		if (cookie == NULL) {
+			return NULL;
+		}
+		res = talloc_asprintf(mem_ctx, "%s:%d:%d:%d:%s",
+					LDB_CONTROL_DIRSYNC_NAME,
+					control->critical,
+					rep_control->flags,
+					rep_control->max_attributes,
+					cookie);
+
+		talloc_free(cookie);
+		return res;
+	}
+	if (strcmp(control->oid, LDB_CONTROL_DIRSYNC_EX_OID) == 0) {
+		char *cookie;
+		struct ldb_dirsync_control *rep_control = talloc_get_type(control->data,
+								struct ldb_dirsync_control);
+
+		if (rep_control == NULL) {
+			return NULL;
+		}
+		cookie = ldb_base64_encode(mem_ctx, rep_control->cookie,
+				rep_control->cookie_len);
+		if (cookie == NULL) {
+			return NULL;
+		}
+		res = talloc_asprintf(mem_ctx, "%s:%d:%d:%d:%s",
+					LDB_CONTROL_DIRSYNC_EX_NAME,
+					control->critical,
+					rep_control->flags,
+					rep_control->max_attributes,
+					cookie);
+
+		talloc_free(cookie);
+		return res;
+	}
+
+	if (strcmp(control->oid, LDB_CONTROL_VERIFY_NAME_OID) == 0) {
+		struct ldb_verify_name_control *rep_control = talloc_get_type(control->data, struct ldb_verify_name_control);
+
+		if (rep_control == NULL) {
+			return NULL;
+		}
+		if (rep_control->gc != NULL) {
+			res = talloc_asprintf(mem_ctx, "%s:%d:%d:%s",
+						LDB_CONTROL_VERIFY_NAME_NAME,
+						control->critical,
+						rep_control->flags,
+						rep_control->gc);
+
+		} else {
+			res = talloc_asprintf(mem_ctx, "%s:%d:%d",
+						LDB_CONTROL_VERIFY_NAME_NAME,
+						control->critical,
+						rep_control->flags);
+		}
+		return res;
+	}
+
+	/*
+	 * From here we don't know the control
+	 */
+	if (control->data == NULL) {
+		/*
+		 * We don't know the control but there is no real data attached
+		 * to it so we can represent it with local_oid:oid:criticity.
+		 */
+		res = talloc_asprintf(mem_ctx, "local_oid:%s:%d",
+					control->oid,
+					control->critical);
+	} else {
+		res = talloc_asprintf(mem_ctx, "unknown oid:%s",
+					control->oid);
+	}
+	return res;
+}
+
+
+/*
+ * A little trick to allow one to use constants defined in headers rather than
+ * hardwritten in the file.
+ * "sizeof" will return the \0 char as well so it will take the place of ":"
+ * in the length of the string.
+ */
+#define LDB_CONTROL_CMP(control, NAME) strncmp(control, NAME ":", sizeof(NAME))
+
+/* Parse one string and return associated control if parsing is successful*/
+struct ldb_control *ldb_parse_control_from_string(struct ldb_context *ldb, TALLOC_CTX *mem_ctx, const char *control_strings)
+{
+	struct ldb_control *ctrl;
+
+	if (!(ctrl = talloc(mem_ctx, struct ldb_control))) {
+		ldb_oom(ldb);
+		return NULL;
+	}
+
+	if (LDB_CONTROL_CMP(control_strings,
+				LDB_CONTROL_VLV_REQ_NAME) == 0) {
+		struct ldb_vlv_req_control *control;
+		const char *p;
+		char attr[1024];
+		char ctxid[1024];
+		int crit, bc, ac, os, cc, ret;
+
+		attr[0] = '\0';
+		ctxid[0] = '\0';
+		p = &(control_strings[sizeof(LDB_CONTROL_VLV_REQ_NAME)]);
+		ret = sscanf(p, "%d:%d:%d:%d:%d:%1023[^$]", &crit, &bc, &ac, &os, &cc, ctxid);
+		/* We allow 2 ways to encode the GT_EQ case, because the
+		   comparison string might contain null bytes or colons, which
+		   would break sscanf (or indeed any parsing mechanism). */
+		if (ret == 3) {
+			ret = sscanf(p, "%d:%d:%d:>=%1023[^:]:%1023[^$]", &crit, &bc, &ac, attr, ctxid);
+		}
+		if (ret == 3) {
+			int len;
+			ret = sscanf(p, "%d:%d:%d:base64>=%1023[^:]:%1023[^$]", &crit, &bc, &ac, attr, ctxid);
+			len = ldb_base64_decode(attr);
+			if (len < 0) {
+				ret = -1;
+			}
+		}
+
+		if ((ret < 4) || (crit < 0) || (crit > 1)) {
+			ldb_set_errstring(ldb,
+					  "invalid VLV control syntax\n"
+					  " syntax: crit(b):bc(n):ac(n):"
+					  "{os(n):cc(n)|>=val(s)|base64>=val(o)}[:ctxid(o)]\n"
+					  "   note: b = boolean, n = number, s = string, o = b64 binary blob");
+			talloc_free(ctrl);
+			return NULL;
+		}
+		ctrl->oid = LDB_CONTROL_VLV_REQ_OID;
+		ctrl->critical = crit;
+		if (!(control = talloc(ctrl,
+					struct ldb_vlv_req_control))) {
+			ldb_oom(ldb);
+			talloc_free(ctrl);
+			return NULL;
+		}
+		control->beforeCount = bc;
+		control->afterCount = ac;
+		if (attr[0]) {
+			control->type = 1;
+			control->match.gtOrEq.value = talloc_strdup(control, attr);
+			control->match.gtOrEq.value_len = strlen(attr);
+		} else {
+			control->type = 0;
+			control->match.byOffset.offset = os;
+			control->match.byOffset.contentCount = cc;
+		}
+		if (ctxid[0]) {
+			int len = ldb_base64_decode(ctxid);
+			if (len < 0) {
+				ldb_set_errstring(ldb,
+						  "invalid VLV context_id\n");
+				talloc_free(ctrl);
+				return NULL;
+			}
+			control->ctxid_len = len;
+			control->contextId = talloc_memdup(control, ctxid,
+							   control->ctxid_len);
+			if (control->contextId == NULL) {
+				ldb_oom(ldb);
+				talloc_free(ctrl);
+				return NULL;
+			}
+		} else {
+			control->ctxid_len = 0;
+			control->contextId = NULL;
+		}
+		ctrl->data = control;
+
+		return ctrl;
+	}
+
+	if (LDB_CONTROL_CMP(control_strings, LDB_CONTROL_DIRSYNC_NAME) == 0) {
+		struct ldb_dirsync_control *control;
+		const char *p;
+		char *cookie = NULL;
+		int crit, max_attrs, ret;
+		uint32_t flags;
+
+		cookie = talloc_zero_array(ctrl, char,
+					   strlen(control_strings) + 1);
+		if (cookie == NULL) {
+			ldb_oom(ldb);
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		p = &(control_strings[sizeof(LDB_CONTROL_DIRSYNC_NAME)]);
+		ret = sscanf(p, "%d:%u:%d:%[^$]", &crit, &flags, &max_attrs, cookie);
+
+		if ((ret < 3) || (crit < 0) || (crit > 1) || (max_attrs < 0)) {
+			ldb_set_errstring(ldb,
+					  "invalid dirsync control syntax\n"
+					  " syntax: crit(b):flags(n):max_attrs(n)[:cookie(o)]\n"
+					  "   note: b = boolean, n = number, o = b64 binary blob");
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		/* w2k3 seems to ignore the parameter,
+		 * but w2k sends a wrong cookie when this value is to small
+		 * this would cause looping forever, while getting
+		 * the same data and same cookie forever
+		 */
+		if (max_attrs == 0) max_attrs = 0x0FFFFFFF;
+
+		ctrl->oid = LDB_CONTROL_DIRSYNC_OID;
+		ctrl->critical = crit;
+		control = talloc(ctrl, struct ldb_dirsync_control);
+		if (control == NULL) {
+			ldb_oom(ldb);
+			talloc_free(ctrl);
+			return NULL;
+		}
+		control->flags = flags;
+		control->max_attributes = max_attrs;
+		if (*cookie) {
+			int len = ldb_base64_decode(cookie);
+			if (len < 0) {
+				ldb_set_errstring(ldb,
+						  "invalid dirsync cookie\n");
+				talloc_free(ctrl);
+				return NULL;
+			}
+			control->cookie_len = len;
+			control->cookie = (char *)talloc_memdup(control, cookie, control->cookie_len);
+			if (control->cookie == NULL) {
+				ldb_oom(ldb);
+				talloc_free(ctrl);
+				return NULL;
+			}
+		} else {
+			control->cookie = NULL;
+			control->cookie_len = 0;
+		}
+		ctrl->data = control;
+		TALLOC_FREE(cookie);
+
+		return ctrl;
+	}
+	if (LDB_CONTROL_CMP(control_strings, LDB_CONTROL_DIRSYNC_EX_NAME) == 0) {
+		struct ldb_dirsync_control *control;
+		const char *p;
+		char *cookie = NULL;
+		int crit, max_attrs, ret;
+		uint32_t flags;
+
+		cookie = talloc_zero_array(ctrl, char,
+					   strlen(control_strings) + 1);
+		if (cookie == NULL) {
+			ldb_oom(ldb);
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		p = &(control_strings[sizeof(LDB_CONTROL_DIRSYNC_EX_NAME)]);
+		ret = sscanf(p, "%d:%u:%d:%1023[^$]", &crit, &flags, &max_attrs, cookie);
+
+		if ((ret < 3) || (crit < 0) || (crit > 1) || (max_attrs < 0)) {
+			ldb_set_errstring(ldb,
+					  "invalid dirsync_ex control syntax\n"
+					  " syntax: crit(b):flags(n):max_attrs(n)[:cookie(o)]\n"
+					  "   note: b = boolean, n = number, o = b64 binary blob");
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		/* w2k3 seems to ignore the parameter,
+		 * but w2k sends a wrong cookie when this value is to small
+		 * this would cause looping forever, while getting
+		 * the same data and same cookie forever
+		 */
+		if (max_attrs == 0) max_attrs = 0x0FFFFFFF;
+
+		ctrl->oid = LDB_CONTROL_DIRSYNC_EX_OID;
+		ctrl->critical = crit;
+		control = talloc(ctrl, struct ldb_dirsync_control);
+		if (control == NULL) {
+			ldb_oom(ldb);
+			talloc_free(ctrl);
+			return NULL;
+		}
+		control->flags = flags;
+		control->max_attributes = max_attrs;
+		if (*cookie) {
+			int len = ldb_base64_decode(cookie);
+			if (len < 0) {
+				ldb_set_errstring(ldb,
+						  "invalid dirsync_ex cookie"
+						  " (probably too long)\n");
+				talloc_free(ctrl);
+				return NULL;
+			}
+			control->cookie_len = len;
+			control->cookie = (char *)talloc_memdup(control, cookie, control->cookie_len);
+			if (control->cookie == NULL) {
+				ldb_oom(ldb);
+				talloc_free(ctrl);
+				return NULL;
+			}
+		} else {
+			control->cookie = NULL;
+			control->cookie_len = 0;
+		}
+		ctrl->data = control;
+		TALLOC_FREE(cookie);
+
+		return ctrl;
+	}
+
+	if (LDB_CONTROL_CMP(control_strings, LDB_CONTROL_ASQ_NAME) == 0) {
+		struct ldb_asq_control *control;
+		const char *p;
+		char attr[256];
+		int crit, ret;
+
+		attr[0] = '\0';
+		p = &(control_strings[sizeof(LDB_CONTROL_ASQ_NAME)]);
+		ret = sscanf(p, "%d:%255[^$]", &crit, attr);
+		if ((ret != 2) || (crit < 0) || (crit > 1) || (attr[0] == '\0')) {
+			ldb_set_errstring(ldb,
+					  "invalid asq control syntax\n"
+					  " syntax: crit(b):attr(s)\n"
+					  "   note: b = boolean, s = string");
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		ctrl->oid = LDB_CONTROL_ASQ_OID;
+		ctrl->critical = crit;
+		control = talloc(ctrl, struct ldb_asq_control);
+		if (control == NULL) {
+			ldb_oom(ldb);
+			talloc_free(ctrl);
+			return NULL;
+		}
+		control->request = 1;
+		control->source_attribute = talloc_strdup(control, attr);
+		control->src_attr_len = strlen(attr);
+		ctrl->data = control;
+
+		return ctrl;
+	}
+
+	if (LDB_CONTROL_CMP(control_strings, LDB_CONTROL_EXTENDED_DN_NAME) == 0) {
+		struct ldb_extended_dn_control *control;
+		const char *p;
+		int crit, type, ret;
+
+		p = &(control_strings[sizeof(LDB_CONTROL_EXTENDED_DN_NAME)]);
+		ret = sscanf(p, "%d:%d", &crit, &type);
+		if ((ret != 2) || (crit < 0) || (crit > 1) || (type < 0) || (type > 1)) {
+			ret = sscanf(p, "%d", &crit);
+			if ((ret != 1) || (crit < 0) || (crit > 1)) {
+				ldb_set_errstring(ldb,
+						  "invalid extended_dn control syntax\n"
+						  " syntax: crit(b)[:type(i)]\n"
+						  "   note: b = boolean\n"
+						  "         i = integer\n"
+						  "   valid values are: 0 - hexadecimal representation\n"
+						  "                     1 - normal string representation");
+				talloc_free(ctrl);
+				return NULL;
+			}
+			control = NULL;
+		} else {
+			control = talloc(ctrl, struct ldb_extended_dn_control);
+			if (control == NULL) {
+				ldb_oom(ldb);
+				talloc_free(ctrl);
+				return NULL;
+			}
+			control->type = type;
+		}
+
+		ctrl->oid = LDB_CONTROL_EXTENDED_DN_OID;
+		ctrl->critical = crit;
+		ctrl->data = control;
+
+		return ctrl;
+	}
+
+	if (LDB_CONTROL_CMP(control_strings, LDB_CONTROL_SD_FLAGS_NAME) == 0) {
+		struct ldb_sd_flags_control *control;
+		const char *p;
+		int crit, ret;
+		unsigned secinfo_flags;
+
+		p = &(control_strings[sizeof(LDB_CONTROL_SD_FLAGS_NAME)]);
+		ret = sscanf(p, "%d:%u", &crit, &secinfo_flags);
+		if ((ret != 2) || (crit < 0) || (crit > 1) || (secinfo_flags > 0xF)) {
+			ldb_set_errstring(ldb,
+					  "invalid sd_flags control syntax\n"
+					  " syntax: crit(b):secinfo_flags(n)\n"
+					  "   note: b = boolean, n = number");
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		ctrl->oid = LDB_CONTROL_SD_FLAGS_OID;
+		ctrl->critical = crit;
+		control = talloc(ctrl, struct ldb_sd_flags_control);
+		if (control == NULL) {
+			ldb_oom(ldb);
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		control->secinfo_flags = secinfo_flags;
+		ctrl->data = control;
+
+		return ctrl;
+	}
+
+	if (LDB_CONTROL_CMP(control_strings, LDB_CONTROL_SEARCH_OPTIONS_NAME) == 0) {
+		struct ldb_search_options_control *control;
+		const char *p;
+		int crit, ret;
+		unsigned search_options;
+
+		p = &(control_strings[sizeof(LDB_CONTROL_SEARCH_OPTIONS_NAME)]);
+		ret = sscanf(p, "%d:%u", &crit, &search_options);
+		if ((ret != 2) || (crit < 0) || (crit > 1) || (search_options > 0xF)) {
+			ldb_set_errstring(ldb,
+					  "invalid search_options control syntax\n"
+					  " syntax: crit(b):search_options(n)\n"
+					  "   note: b = boolean, n = number");
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		ctrl->oid = LDB_CONTROL_SEARCH_OPTIONS_OID;
+		ctrl->critical = crit;
+		control = talloc(ctrl, struct ldb_search_options_control);
+		if (control == NULL) {
+			ldb_oom(ldb);
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		control->search_options = search_options;
+		ctrl->data = control;
+
+		return ctrl;
+	}
+
+	if (LDB_CONTROL_CMP(control_strings, LDB_CONTROL_BYPASS_OPERATIONAL_NAME) == 0) {
+		const char *p;
+		int crit, ret;
+
+		p = &(control_strings[sizeof(LDB_CONTROL_BYPASS_OPERATIONAL_NAME)]);
+		ret = sscanf(p, "%d", &crit);
+		if ((ret != 1) || (crit < 0) || (crit > 1)) {
+			ldb_set_errstring(ldb,
+					  "invalid bypassoperational control syntax\n"
+					  " syntax: crit(b)\n"
+					  "   note: b = boolean");
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		ctrl->oid = LDB_CONTROL_BYPASS_OPERATIONAL_OID;
+		ctrl->critical = crit;
+		ctrl->data = NULL;
+
+		return ctrl;
+	}
+
+	if (LDB_CONTROL_CMP(control_strings, LDB_CONTROL_RELAX_NAME) == 0) {
+		const char *p;
+		int crit, ret;
+
+		p = &(control_strings[sizeof(LDB_CONTROL_RELAX_NAME)]);
+		ret = sscanf(p, "%d", &crit);
+		if ((ret != 1) || (crit < 0) || (crit > 1)) {
+			ldb_set_errstring(ldb,
+					  "invalid relax control syntax\n"
+					  " syntax: crit(b)\n"
+					  "   note: b = boolean");
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		ctrl->oid = LDB_CONTROL_RELAX_OID;
+		ctrl->critical = crit;
+		ctrl->data = NULL;
+
+		return ctrl;
+	}
+
+	if (LDB_CONTROL_CMP(control_strings, LDB_CONTROL_RECALCULATE_SD_NAME) == 0) {
+		const char *p;
+		int crit, ret;
+
+		p = &(control_strings[sizeof(LDB_CONTROL_RECALCULATE_SD_NAME)]);
+		ret = sscanf(p, "%d", &crit);
+		if ((ret != 1) || (crit < 0) || (crit > 1)) {
+			ldb_set_errstring(ldb,
+					  "invalid recalculate_sd control syntax\n"
+					  " syntax: crit(b)\n"
+					  "   note: b = boolean");
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		ctrl->oid = LDB_CONTROL_RECALCULATE_SD_OID;
+		ctrl->critical = crit;
+		ctrl->data = NULL;
+
+		return ctrl;
+	}
+
+	if (LDB_CONTROL_CMP(control_strings, LDB_CONTROL_DOMAIN_SCOPE_NAME) == 0) {
+		const char *p;
+		int crit, ret;
+
+		p = &(control_strings[sizeof(LDB_CONTROL_DOMAIN_SCOPE_NAME)]);
+		ret = sscanf(p, "%d", &crit);
+		if ((ret != 1) || (crit < 0) || (crit > 1)) {
+			ldb_set_errstring(ldb,
+					  "invalid domain_scope control syntax\n"
+					  " syntax: crit(b)\n"
+					  "   note: b = boolean");
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		ctrl->oid = LDB_CONTROL_DOMAIN_SCOPE_OID;
+		ctrl->critical = crit;
+		ctrl->data = NULL;
+
+		return ctrl;
+	}
+
+	if (LDB_CONTROL_CMP(control_strings, LDB_CONTROL_PAGED_RESULTS_NAME) == 0) {
+		struct ldb_paged_control *control;
+		const char *p;
+		char cookie[1024];
+		int crit, size, ret;
+
+		cookie[0] = '\0';
+		p = &(control_strings[sizeof(LDB_CONTROL_PAGED_RESULTS_NAME)]);
+		ret = sscanf(p, "%d:%d:%1023[^$]", &crit, &size, cookie);
+		if ((ret < 2) || (ret > 3) || (crit < 0) || (crit > 1) ||
+		    (size < 0)) {
+			ldb_set_errstring(ldb,
+				"invalid paged_results control syntax\n"
+				" syntax: crit(b):size(n)[:cookie(base64)]\n"
+				"   note: b = boolean, n = number");
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		ctrl->oid = LDB_CONTROL_PAGED_RESULTS_OID;
+		ctrl->critical = crit;
+		control = talloc(ctrl, struct ldb_paged_control);
+		if (control == NULL) {
+			ldb_oom(ldb);
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		control->size = size;
+		if (cookie[0] != '\0') {
+			int len = ldb_base64_decode(cookie);
+			if (len < 0) {
+				ldb_set_errstring(ldb,
+						  "invalid paged_results cookie"
+						  " (probably too long)\n");
+				talloc_free(ctrl);
+				return NULL;
+			}
+			control->cookie_len = len;
+			control->cookie = talloc_memdup(control, cookie, control->cookie_len);
+			if (control->cookie == NULL) {
+				ldb_oom(ldb);
+				talloc_free(ctrl);
+				return NULL;
+			}
+		} else {
+			control->cookie = NULL;
+			control->cookie_len = 0;
+		}
+		ctrl->data = control;
+
+		return ctrl;
+	}
+
+	if (LDB_CONTROL_CMP(control_strings, LDB_CONTROL_SERVER_SORT_NAME) == 0) {
+		struct ldb_server_sort_control **control;
+		const char *p;
+		char attr[256];
+		char rule[128];
+		int crit, rev, ret;
+
+		attr[0] = '\0';
+		rule[0] = '\0';
+		p = &(control_strings[sizeof(LDB_CONTROL_SERVER_SORT_NAME)]);
+		ret = sscanf(p, "%d:%d:%255[^:]:%127[^:]", &crit, &rev, attr, rule);
+		if ((ret < 3) || (crit < 0) || (crit > 1) || (rev < 0 ) || (rev > 1) ||attr[0] == '\0') {
+			ldb_set_errstring(ldb,
+					  "invalid server_sort control syntax\n"
+					  " syntax: crit(b):rev(b):attr(s)[:rule(s)]\n"
+					  "   note: b = boolean, s = string");
+			talloc_free(ctrl);
+			return NULL;
+		}
+		ctrl->oid = LDB_CONTROL_SERVER_SORT_OID;
+		ctrl->critical = crit;
+		control = talloc_array(ctrl, struct ldb_server_sort_control *, 2);
+		if (control == NULL) {
+			ldb_oom(ldb);
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		control[0] = talloc(control, struct ldb_server_sort_control);
+		if (control[0] == NULL) {
+			ldb_oom(ldb);
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		control[0]->attributeName = talloc_strdup(control, attr);
+		if (control[0]->attributeName == NULL) {
+			ldb_oom(ldb);
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		if (rule[0]) {
+			control[0]->orderingRule = talloc_strdup(control, rule);
+			if (control[0]->orderingRule == NULL) {
+				ldb_oom(ldb);
+				talloc_free(ctrl);
+				return NULL;
+			}
+		} else {
+			control[0]->orderingRule = NULL;
+		}
+		control[0]->reverse = rev;
+		control[1] = NULL;
+		ctrl->data = control;
+
+		return ctrl;
+	}
+
+	if (LDB_CONTROL_CMP(control_strings, LDB_CONTROL_NOTIFICATION_NAME) == 0) {
+		const char *p;
+		int crit, ret;
+
+		p = &(control_strings[sizeof(LDB_CONTROL_NOTIFICATION_NAME)]);
+		ret = sscanf(p, "%d", &crit);
+		if ((ret != 1) || (crit < 0) || (crit > 1)) {
+			ldb_set_errstring(ldb,
+					  "invalid notification control syntax\n"
+					  " syntax: crit(b)\n"
+					  "   note: b = boolean");
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		ctrl->oid = LDB_CONTROL_NOTIFICATION_OID;
+		ctrl->critical = crit;
+		ctrl->data = NULL;
+
+		return ctrl;
+	}
+
+	if (LDB_CONTROL_CMP(control_strings, LDB_CONTROL_TREE_DELETE_NAME) == 0) {
+		const char *p;
+		int crit, ret;
+
+		p = &(control_strings[sizeof(LDB_CONTROL_TREE_DELETE_NAME)]);
+		ret = sscanf(p, "%d", &crit);
+		if ((ret != 1) || (crit < 0) || (crit > 1)) {
+			ldb_set_errstring(ldb,
+					  "invalid tree_delete control syntax\n"
+					  " syntax: crit(b)\n"
+					  "   note: b = boolean");
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		ctrl->oid = LDB_CONTROL_TREE_DELETE_OID;
+		ctrl->critical = crit;
+		ctrl->data = NULL;
+
+		return ctrl;
+	}
+
+	if (LDB_CONTROL_CMP(control_strings, LDB_CONTROL_SHOW_DELETED_NAME) == 0) {
+		const char *p;
+		int crit, ret;
+
+		p = &(control_strings[sizeof(LDB_CONTROL_SHOW_DELETED_NAME)]);
+		ret = sscanf(p, "%d", &crit);
+		if ((ret != 1) || (crit < 0) || (crit > 1)) {
+			ldb_set_errstring(ldb,
+					  "invalid show_deleted control syntax\n"
+					  " syntax: crit(b)\n"
+					  "   note: b = boolean");
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		ctrl->oid = LDB_CONTROL_SHOW_DELETED_OID;
+		ctrl->critical = crit;
+		ctrl->data = NULL;
+
+		return ctrl;
+	}
+
+	if (LDB_CONTROL_CMP(control_strings, LDB_CONTROL_SHOW_DEACTIVATED_LINK_NAME) == 0) {
+		const char *p;
+		int crit, ret;
+
+		p = &(control_strings[sizeof(LDB_CONTROL_SHOW_DEACTIVATED_LINK_NAME)]);
+		ret = sscanf(p, "%d", &crit);
+		if ((ret != 1) || (crit < 0) || (crit > 1)) {
+			ldb_set_errstring(ldb,
+					  "invalid show_deactivated_link control syntax\n"
+					  " syntax: crit(b)\n"
+					  "   note: b = boolean");
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		ctrl->oid = LDB_CONTROL_SHOW_DEACTIVATED_LINK_OID;
+		ctrl->critical = crit;
+		ctrl->data = NULL;
+
+		return ctrl;
+	}
+
+	if (LDB_CONTROL_CMP(control_strings, LDB_CONTROL_SHOW_RECYCLED_NAME) == 0) {
+		const char *p;
+		int crit, ret;
+
+		p = &(control_strings[sizeof(LDB_CONTROL_SHOW_RECYCLED_NAME)]);
+		ret = sscanf(p, "%d", &crit);
+		if ((ret != 1) || (crit < 0) || (crit > 1)) {
+			ldb_set_errstring(ldb,
+					  "invalid show_recycled control syntax\n"
+					  " syntax: crit(b)\n"
+					  "   note: b = boolean");
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		ctrl->oid = LDB_CONTROL_SHOW_RECYCLED_OID;
+		ctrl->critical = crit;
+		ctrl->data = NULL;
+
+		return ctrl;
+	}
+
+	if (LDB_CONTROL_CMP(control_strings, LDB_CONTROL_PERMISSIVE_MODIFY_NAME) == 0) {
+		const char *p;
+		int crit, ret;
+
+		p = &(control_strings[sizeof(LDB_CONTROL_PERMISSIVE_MODIFY_NAME)]);
+		ret = sscanf(p, "%d", &crit);
+		if ((ret != 1) || (crit < 0) || (crit > 1)) {
+			ldb_set_errstring(ldb,
+					  "invalid permissive_modify control syntax\n"
+					  " syntax: crit(b)\n"
+					  "   note: b = boolean");
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		ctrl->oid = LDB_CONTROL_PERMISSIVE_MODIFY_OID;
+		ctrl->critical = crit;
+		ctrl->data = NULL;
+
+		return ctrl;
+	}
+
+	if (LDB_CONTROL_CMP(control_strings, LDB_CONTROL_REVEAL_INTERNALS_NAME) == 0) {
+		const char *p;
+		int crit, ret;
+
+		p = &(control_strings[sizeof(LDB_CONTROL_REVEAL_INTERNALS_NAME)]);
+		ret = sscanf(p, "%d", &crit);
+		if ((ret != 1) || (crit < 0) || (crit > 1)) {
+			ldb_set_errstring(ldb,
+					  "invalid reveal_internals control syntax\n"
+					  " syntax: crit(b)\n"
+					  "   note: b = boolean");
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		ctrl->oid = LDB_CONTROL_REVEAL_INTERNALS;
+		ctrl->critical = crit;
+		ctrl->data = NULL;
+
+		return ctrl;
+	}
+
+	if (strncmp(control_strings, "local_oid:", 10) == 0) {
+		const char *p;
+		int crit = 0, ret = 0;
+		char oid[256];
+
+		oid[0] = '\0';
+		p = &(control_strings[10]);
+		ret = sscanf(p, "%255[^:]:%d", oid, &crit);
+
+		if ((ret != 2) || strlen(oid) == 0 || (crit < 0) || (crit > 1)) {
+			ldb_set_errstring(ldb,
+					  "invalid local_oid control syntax\n"
+					  " syntax: oid(s):crit(b)\n"
+					  "   note: b = boolean, s = string");
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		ctrl->oid = talloc_strdup(ctrl, oid);
+		if (!ctrl->oid) {
+			ldb_oom(ldb);
+			talloc_free(ctrl);
+			return NULL;
+		}
+		ctrl->critical = crit;
+		ctrl->data = NULL;
+
+		return ctrl;
+	}
+
+	if (LDB_CONTROL_CMP(control_strings, LDB_CONTROL_RODC_DCPROMO_NAME) == 0) {
+		const char *p;
+		int crit, ret;
+
+		p = &(control_strings[sizeof(LDB_CONTROL_RODC_DCPROMO_NAME)]);
+		ret = sscanf(p, "%d", &crit);
+		if ((ret != 1) || (crit < 0) || (crit > 1)) {
+			ldb_set_errstring(ldb,
+					  "invalid rodc_join control syntax\n"
+					  " syntax: crit(b)\n"
+					  "   note: b = boolean");
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		ctrl->oid = LDB_CONTROL_RODC_DCPROMO_OID;
+		ctrl->critical = crit;
+		ctrl->data = NULL;
+
+		return ctrl;
+	}
+
+	if (LDB_CONTROL_CMP(control_strings, LDB_CONTROL_PROVISION_NAME) == 0) {
+		const char *p;
+		int crit, ret;
+
+		p = &(control_strings[sizeof(LDB_CONTROL_PROVISION_NAME)]);
+		ret = sscanf(p, "%d", &crit);
+		if ((ret != 1) || (crit < 0) || (crit > 1)) {
+			ldb_set_errstring(ldb,
+					  "invalid provision control syntax\n"
+					  " syntax: crit(b)\n"
+					  "   note: b = boolean");
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		ctrl->oid = LDB_CONTROL_PROVISION_OID;
+		ctrl->critical = crit;
+		ctrl->data = NULL;
+
+		return ctrl;
+	}
+	if (LDB_CONTROL_CMP(control_strings, LDB_CONTROL_VERIFY_NAME_NAME) == 0) {
+		const char *p;
+		char gc[1024];
+		int crit, flags, ret;
+		struct ldb_verify_name_control *control;
+
+		gc[0] = '\0';
+
+		p = &(control_strings[sizeof(LDB_CONTROL_VERIFY_NAME_NAME)]);
+		ret = sscanf(p, "%d:%d:%1023[^$]", &crit, &flags, gc);
+		if ((ret != 3) || (crit < 0) || (crit > 1)) {
+			ret = sscanf(p, "%d:%d", &crit, &flags);
+			if ((ret != 2) || (crit < 0) || (crit > 1)) {
+				ldb_set_errstring(ldb,
+						  "invalid verify_name control syntax\n"
+						  " syntax: crit(b):flags(i)[:gc(s)]\n"
+						  "   note: b = boolean"
+						  "   note: i = integer"
+						  "   note: s = string");
+				talloc_free(ctrl);
+				return NULL;
+			}
+		}
+
+		ctrl->oid = LDB_CONTROL_VERIFY_NAME_OID;
+		ctrl->critical = crit;
+		control = talloc(ctrl, struct ldb_verify_name_control);
+		if (control == NULL) {
+			ldb_oom(ldb);
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		control->gc = talloc_strdup(control, gc);
+		if (control->gc == NULL) {
+			ldb_oom(ldb);
+			talloc_free(ctrl);
+			return NULL;
+		}
+
+		control->gc_len = strlen(gc);
+		control->flags = flags;
+		ctrl->data = control;
+		return ctrl;
+	}
+	/*
+	 * When no matching control has been found.
+	 */
+	TALLOC_FREE(ctrl);
+	return NULL;
+}
+
+/* Parse controls from the format used on the command line and in ejs */
+struct ldb_control **ldb_parse_control_strings(struct ldb_context *ldb, TALLOC_CTX *mem_ctx, const char **control_strings)
+{
+	unsigned int i;
+	struct ldb_control **ctrl;
+
+	if (control_strings == NULL || control_strings[0] == NULL)
+		return NULL;
+
+	for (i = 0; control_strings[i]; i++);
+
+	ctrl = talloc_array(mem_ctx, struct ldb_control *, i + 1);
+
+	ldb_reset_err_string(ldb);
+	for (i = 0; control_strings[i]; i++) {
+		ctrl[i] = ldb_parse_control_from_string(ldb, ctrl, control_strings[i]);
+		if (ctrl[i] == NULL) {
+			if (ldb_errstring(ldb) == NULL) {
+				/* no controls matched, throw an error */
+				ldb_asprintf_errstring(ldb, "Invalid control name: '%s'", control_strings[i]);
+			}
+			talloc_free(ctrl);
+			return NULL;
+		}
+	}
+
+	ctrl[i] = NULL;
+
+	return ctrl;
+}
+
+
diff --git a/lib/ldb/common/ldb_debug.c b/lib/ldb/common/ldb_debug.c
new file mode 100644
index 0000000..d5e9e7a
--- /dev/null
+++ b/lib/ldb/common/ldb_debug.c
@@ -0,0 +1,150 @@
+/* 
+   ldb database library
+
+   Copyright (C) Andrew Tridgell  2004
+
+     ** NOTE! The following LGPL license applies to the ldb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+   
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+/*
+ *  Name: ldb
+ *
+ *  Component: ldb debug
+ *
+ *  Description: functions for printing debug messages
+ *
+ *  Author: Andrew Tridgell
+ */
+
+#include "ldb_private.h"
+
+/*
+  this allows the user to choose their own debug function
+*/
+int ldb_set_debug(struct ldb_context *ldb,
+		  void (*debug)(void *context, enum ldb_debug_level level, 
+				const char *fmt, va_list ap),
+		  void *context)
+{
+	ldb->debug_ops.debug = debug;
+	ldb->debug_ops.context = context;
+	return 0;
+}
+
+/*
+  debug function for ldb_set_debug_stderr
+*/
+static void ldb_debug_stderr(void *context, enum ldb_debug_level level, 
+			     const char *fmt, va_list ap) PRINTF_ATTRIBUTE(3,0);
+static void ldb_debug_stderr(void *context, enum ldb_debug_level level, 
+			     const char *fmt, va_list ap)
+{
+	if (level <= LDB_DEBUG_WARNING) {
+		vfprintf(stderr, fmt, ap);
+		fprintf(stderr, "\n");
+	}
+}
+
+static void ldb_debug_stderr_all(void *context, enum ldb_debug_level level, 
+			     const char *fmt, va_list ap) PRINTF_ATTRIBUTE(3,0);
+static void ldb_debug_stderr_all(void *context, enum ldb_debug_level level, 
+			     const char *fmt, va_list ap)
+{
+	vfprintf(stderr, fmt, ap);
+	fprintf(stderr, "\n");
+}
+
+/*
+  convenience function to setup debug messages on stderr
+  messages of level LDB_DEBUG_WARNING and higher are printed
+*/
+int ldb_set_debug_stderr(struct ldb_context *ldb)
+{
+	return ldb_set_debug(ldb, ldb_debug_stderr, ldb);
+}
+
+/*
+  log a message (va_list helper for ldb_tevent_debug)
+*/
+void ldb_vdebug(struct ldb_context *ldb, enum ldb_debug_level level, const char *fmt, va_list ap)
+{
+	if (ldb->debug_ops.debug == NULL) {
+		if (ldb->flags & LDB_FLG_ENABLE_TRACING) {
+			ldb_set_debug(ldb, ldb_debug_stderr_all, ldb);
+		} else {
+			ldb_set_debug_stderr(ldb);
+		}
+	}
+	ldb->debug_ops.debug(ldb->debug_ops.context, level, fmt, ap);
+}
+
+/*
+  log a message
+*/
+void ldb_debug(struct ldb_context *ldb, enum ldb_debug_level level, const char *fmt, ...)
+{
+	va_list ap;
+	va_start(ap, fmt);
+	ldb_vdebug(ldb, level, fmt, ap);
+	va_end(ap);
+}
+
+/*
+  add to an accumulated log message
+ */
+void ldb_debug_add(struct ldb_context *ldb, const char *fmt, ...)
+{
+	va_list ap;
+	va_start(ap, fmt);
+	if (ldb->partial_debug == NULL) {
+		ldb->partial_debug = talloc_vasprintf(ldb, fmt, ap);
+	} else {
+		ldb->partial_debug = talloc_vasprintf_append(ldb->partial_debug, 
+							     fmt, ap);
+	}
+	va_end(ap);
+}
+
+/*
+  send the accumulated log message, and free it
+ */
+void ldb_debug_end(struct ldb_context *ldb, enum ldb_debug_level level)
+{
+	ldb_debug(ldb, level, "%s", ldb->partial_debug);
+	talloc_free(ldb->partial_debug);
+	ldb->partial_debug = NULL;
+}
+
+/*
+  log a message, and set the ldb error string to the same message
+*/
+void ldb_debug_set(struct ldb_context *ldb, enum ldb_debug_level level, 
+		   const char *fmt, ...)
+{
+	va_list ap;
+	char *msg;
+	va_start(ap, fmt);
+	msg = talloc_vasprintf(ldb, fmt, ap);
+	va_end(ap);
+	if (msg != NULL) {
+		ldb_set_errstring(ldb, msg);
+		ldb_debug(ldb, level, "%s", msg);
+	}
+	talloc_free(msg);
+}
+
diff --git a/lib/ldb/common/ldb_dn.c b/lib/ldb/common/ldb_dn.c
new file mode 100644
index 0000000..fb7a95b
--- /dev/null
+++ b/lib/ldb/common/ldb_dn.c
@@ -0,0 +1,2238 @@
+/*
+   ldb database library
+
+   Copyright (C) Simo Sorce 2005
+
+     ** NOTE! The following LGPL license applies to the ldb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+/*
+ *  Name: ldb
+ *
+ *  Component: ldb dn creation and manipulation utility functions
+ *
+ *  Description: - explode a dn into it's own basic elements
+ *                 and put them in a structure (only if necessary)
+ *               - manipulate ldb_dn structures
+ *
+ *  Author: Simo Sorce
+ */
+
+#include "ldb_private.h"
+#include <ctype.h>
+
+#define LDB_DN_NULL_FAILED(x) if (!(x)) goto failed
+
+#define LDB_FREE(x) do { talloc_free(x); x = NULL; } while(0)
+
+/**
+   internal ldb exploded dn structures
+*/
+struct ldb_dn_component {
+
+	char *name;
+	struct ldb_val value;
+
+	char *cf_name;
+	struct ldb_val cf_value;
+};
+
+struct ldb_dn_ext_component {
+
+	const char *name;
+	struct ldb_val value;
+};
+
+struct ldb_dn {
+
+	struct ldb_context *ldb;
+
+	/* Special DNs are always linearized */
+	bool special;
+	bool invalid;
+
+	bool valid_case;
+
+	char *linearized;
+	char *ext_linearized;
+	char *casefold;
+
+	unsigned int comp_num;
+	struct ldb_dn_component *components;
+
+	unsigned int ext_comp_num;
+	struct ldb_dn_ext_component *ext_components;
+};
+
+/* it is helpful to be able to break on this in gdb */
+static void ldb_dn_mark_invalid(struct ldb_dn *dn)
+{
+	dn->invalid = true;
+}
+
+/* strdn may be NULL */
+struct ldb_dn *ldb_dn_from_ldb_val(TALLOC_CTX *mem_ctx,
+                                   struct ldb_context *ldb,
+                                   const struct ldb_val *strdn)
+{
+	struct ldb_dn *dn;
+
+	if (ldb == NULL || strdn == NULL) {
+		return NULL;
+	}
+	if (strdn->data
+	    && (strnlen((const char*)strdn->data, strdn->length) != strdn->length)) {
+		/* The RDN must not contain a character with value 0x0 */
+		return NULL;
+	}
+
+	dn = talloc_zero(mem_ctx, struct ldb_dn);
+	LDB_DN_NULL_FAILED(dn);
+
+	dn->ldb = talloc_get_type(ldb, struct ldb_context);
+	if (dn->ldb == NULL) {
+		/* the caller probably got the arguments to
+		   ldb_dn_new() mixed up */
+		talloc_free(dn);
+		return NULL;
+	}
+
+	if (strdn->data && strdn->length) {
+		const char *data = (const char *)strdn->data;
+		size_t length = strdn->length;
+
+		if (data[0] == '@') {
+			dn->special = true;
+		}
+		dn->ext_linearized = talloc_strndup(dn, data, length);
+		LDB_DN_NULL_FAILED(dn->ext_linearized);
+
+		if (data[0] == '<') {
+			const char *p_save, *p = dn->ext_linearized;
+			do {
+				p_save = p;
+				p = strstr(p, ">;");
+				if (p) {
+					p = p + 2;
+				}
+			} while (p);
+
+			if (p_save == dn->ext_linearized) {
+				dn->linearized = talloc_strdup(dn, "");
+			} else {
+				dn->linearized = talloc_strdup(dn, p_save);
+			}
+			LDB_DN_NULL_FAILED(dn->linearized);
+		} else {
+			dn->linearized = dn->ext_linearized;
+			dn->ext_linearized = NULL;
+		}
+	} else {
+		dn->linearized = talloc_strdup(dn, "");
+		LDB_DN_NULL_FAILED(dn->linearized);
+	}
+
+	return dn;
+
+failed:
+	talloc_free(dn);
+	return NULL;
+}
+
+/* strdn may be NULL */
+struct ldb_dn *ldb_dn_new(TALLOC_CTX *mem_ctx,
+			  struct ldb_context *ldb,
+			  const char *strdn)
+{
+	struct ldb_val blob;
+	blob.data = discard_const_p(uint8_t, strdn);
+	blob.length = strdn ? strlen(strdn) : 0;
+	return ldb_dn_from_ldb_val(mem_ctx, ldb, &blob);
+}
+
+struct ldb_dn *ldb_dn_new_fmt(TALLOC_CTX *mem_ctx,
+			      struct ldb_context *ldb,
+			      const char *new_fmt, ...)
+{
+	char *strdn;
+	va_list ap;
+
+	if (! ldb) return NULL;
+
+	va_start(ap, new_fmt);
+	strdn = talloc_vasprintf(mem_ctx, new_fmt, ap);
+	va_end(ap);
+
+	if (strdn) {
+		struct ldb_dn *dn = ldb_dn_new(mem_ctx, ldb, strdn);
+		talloc_free(strdn);
+		return dn;
+	}
+
+	return NULL;
+}
+
+/* see RFC2253 section 2.4 */
+static int ldb_dn_escape_internal(char *dst, const char *src, int len)
+{
+	char c;
+	char *d;
+	int i;
+	d = dst;
+
+	for (i = 0; i < len; i++){
+		c = src[i];
+		switch (c) {
+		case ' ':
+			if (i == 0 || i == len - 1) {
+				/* if at the beginning or end
+				 * of the string then escape */
+				*d++ = '\\';
+				*d++ = c;
+			} else {
+				/* otherwise don't escape */
+				*d++ = c;
+			}
+			break;
+
+		case '#':
+			/* despite the RFC, windows escapes a #
+			   anywhere in the string */
+		case ',':
+		case '+':
+		case '"':
+		case '\\':
+		case '<':
+		case '>':
+		case '?':
+			/* these must be escaped using \c form */
+			*d++ = '\\';
+			*d++ = c;
+			break;
+
+		case ';':
+		case '\r':
+		case '\n':
+		case '=':
+		case '\0': {
+			/* any others get \XX form */
+			unsigned char v;
+			const char *hexbytes = "0123456789ABCDEF";
+			v = (const unsigned char)c;
+			*d++ = '\\';
+			*d++ = hexbytes[v>>4];
+			*d++ = hexbytes[v&0xF];
+			break;
+		}
+		default:
+			*d++ = c;
+		}
+	}
+
+	/* return the length of the resulting string */
+	return (d - dst);
+}
+
+char *ldb_dn_escape_value(TALLOC_CTX *mem_ctx, struct ldb_val value)
+{
+	char *dst;
+	size_t len;
+	if (!value.length)
+		return NULL;
+
+	/* allocate destination string, it will be at most 3 times the source */
+	dst = talloc_array(mem_ctx, char, value.length * 3 + 1);
+	if ( ! dst) {
+		talloc_free(dst);
+		return NULL;
+	}
+
+	len = ldb_dn_escape_internal(dst, (const char *)value.data, value.length);
+
+	dst = talloc_realloc(mem_ctx, dst, char, len + 1);
+	if ( ! dst) {
+		talloc_free(dst);
+		return NULL;
+	}
+	dst[len] = '\0';
+	return dst;
+}
+
+/*
+  explode a DN string into a ldb_dn structure
+  based on RFC4514 except that we don't support multiple valued RDNs
+
+  TODO: according to MS-ADTS:3.1.1.5.2 Naming Constraints
+  DN must be compliant with RFC2253
+*/
+static bool ldb_dn_explode(struct ldb_dn *dn)
+{
+	char *p, *ex_name = NULL, *ex_value = NULL, *data, *d, *dt, *t;
+	bool trim = true;
+	bool in_extended = true;
+	bool in_ex_name = false;
+	bool in_ex_value = false;
+	bool in_attr = false;
+	bool in_value = false;
+	bool in_quote = false;
+	bool is_oid = false;
+	bool escape = false;
+	unsigned int x;
+	size_t l = 0;
+	int ret;
+	char *parse_dn;
+	bool is_index;
+
+	if (dn == NULL || dn->invalid) {
+		return false;
+	}
+
+	if (dn->components != NULL) {
+		return true;
+	}
+
+	if (dn->ext_linearized != NULL) {
+		parse_dn = dn->ext_linearized;
+	} else {
+		parse_dn = dn->linearized;
+	}
+
+	if (parse_dn == NULL) {
+		return false;
+	}
+
+	is_index = (strncmp(parse_dn, "DN=@INDEX:", 10) == 0);
+
+	/* Empty DNs */
+	if (parse_dn[0] == '\0') {
+		return true;
+	}
+
+	/* Special DNs case */
+	if (dn->special) {
+		return true;
+	}
+
+	LDB_FREE(dn->ext_components);
+	dn->ext_comp_num = 0;
+	dn->comp_num = 0;
+
+	/* in the common case we have 3 or more components */
+	/* make sure all components are zeroed, other functions depend on it */
+	dn->components = talloc_zero_array(dn, struct ldb_dn_component, 3);
+	if (dn->components == NULL) {
+		return false;
+	}
+
+	/* Components data space is allocated here once */
+	data = talloc_array(dn->components, char, strlen(parse_dn) + 1);
+	if (data == NULL) {
+		goto failed;
+	}
+
+	p = parse_dn;
+	t = NULL;
+	d = dt = data;
+
+	while (*p) {
+		if (in_extended) {
+
+			if (!in_ex_name && !in_ex_value) {
+
+				if (p[0] == '<') {
+					p++;
+					ex_name = d;
+					in_ex_name = true;
+					continue;
+				} else {
+					in_extended = false;
+					in_attr = true;
+					dt = d;
+
+					continue;
+				}
+			}
+
+			if (in_ex_name && *p == '=') {
+				*d++ = '\0';
+				p++;
+				ex_value = d;
+				in_ex_name = false;
+				in_ex_value = true;
+				continue;
+			}
+
+			if (in_ex_value && *p == '>') {
+				struct ldb_dn_ext_component *ext_comp = NULL;
+				const struct ldb_dn_extended_syntax *ext_syntax;
+				struct ldb_val ex_val = {
+					.data = (uint8_t *)ex_value,
+					.length = d - ex_value
+				};
+
+				*d++ = '\0';
+				p++;
+				in_ex_value = false;
+
+				/* Process name and ex_value */
+
+				ext_comp = talloc_realloc(
+					dn,
+					dn->ext_components,
+					struct ldb_dn_ext_component,
+					dn->ext_comp_num + 1);
+
+				if (ext_comp == NULL) {
+					/* ouch ! */
+					goto failed;
+				}
+
+				dn->ext_components = ext_comp;
+
+				ext_syntax = ldb_dn_extended_syntax_by_name(dn->ldb, ex_name);
+				if (ext_syntax == NULL) {
+					/* We don't know about this type of extended DN */
+					goto failed;
+				}
+
+				dn->ext_components[dn->ext_comp_num].name = ext_syntax->name;
+				ret = ext_syntax->read_fn(dn->ldb, dn->ext_components,
+							  &ex_val, &dn->ext_components[dn->ext_comp_num].value);
+				if (ret != LDB_SUCCESS) {
+					ldb_dn_mark_invalid(dn);
+					goto failed;
+				}
+
+				dn->ext_comp_num++;
+
+				if (*p == '\0') {
+					/* We have reached the end (extended component only)! */
+					talloc_free(data);
+					return true;
+
+				} else if (*p == ';') {
+					p++;
+					continue;
+				} else {
+					ldb_dn_mark_invalid(dn);
+					goto failed;
+				}
+			}
+
+			*d++ = *p++;
+			continue;
+		}
+		if (in_attr) {
+			if (trim) {
+				if (*p == ' ') {
+					p++;
+					continue;
+				}
+
+				/* first char */
+				trim = false;
+
+				if (!isascii(*p)) {
+					/* attr names must be ascii only */
+					ldb_dn_mark_invalid(dn);
+					goto failed;
+				}
+
+				if (isdigit(*p)) {
+					is_oid = true;
+				} else
+				if ( ! isalpha(*p)) {
+					/* not a digit nor an alpha,
+ 					 * invalid attribute name */
+					ldb_dn_mark_invalid(dn);
+					goto failed;
+				}
+
+				/* Copy this character across from parse_dn,
+				 * now we have trimmed out spaces */
+				*d++ = *p++;
+				continue;
+			}
+
+			if (*p == ' ') {
+				p++;
+				/* valid only if we are at the end */
+				trim = true;
+				continue;
+			}
+
+			if (*p == '=') {
+				/* attribute terminated */
+				in_attr = false;
+				in_value = true;
+				trim = true;
+				l = 0;
+
+				/* Terminate this string in d
+				 * (which is a copy of parse_dn
+				 *  with spaces trimmed) */
+				*d++ = '\0';
+				dn->components[dn->comp_num].name = talloc_strdup(dn->components, dt);
+				if (dn->components[dn->comp_num].name == NULL) {
+					/* ouch */
+					goto failed;
+				}
+
+				dt = d;
+
+				p++;
+				continue;
+			}
+
+			if (!isascii(*p)) {
+				/* attr names must be ascii only */
+				ldb_dn_mark_invalid(dn);
+				goto failed;
+			}
+
+			if (is_oid && ( ! (isdigit(*p) || (*p == '.')))) {
+				/* not a digit nor a dot,
+				 * invalid attribute oid */
+				ldb_dn_mark_invalid(dn);
+				goto failed;
+			} else
+			if ( ! (isalpha(*p) || isdigit(*p) || (*p == '-'))) {
+				/* not ALPHA, DIGIT or HYPHEN */
+				ldb_dn_mark_invalid(dn);
+				goto failed;
+			}
+
+			*d++ = *p++;
+			continue;
+		}
+
+		if (in_value) {
+			if (in_quote) {
+				if (*p == '\"') {
+					if (p[-1] != '\\') {
+						p++;
+						in_quote = false;
+						continue;
+					}
+				}
+				*d++ = *p++;
+				l++;
+				continue;
+			}
+
+			if (trim) {
+				if (*p == ' ') {
+					p++;
+					continue;
+				}
+
+				/* first char */
+				trim = false;
+
+				if (*p == '\"') {
+					in_quote = true;
+					p++;
+					continue;
+				}
+			}
+
+			switch (*p) {
+
+			/* TODO: support ber encoded values
+			case '#':
+			*/
+
+			case ',':
+				if (escape) {
+					*d++ = *p++;
+					l++;
+					escape = false;
+					continue;
+				}
+				/* ok found value terminator */
+
+				if (t != NULL) {
+					/* trim back */
+					d -= (p - t);
+					l -= (p - t);
+					t = NULL;
+				}
+
+				in_attr = true;
+				in_value = false;
+				trim = true;
+
+				p++;
+				*d++ = '\0';
+
+				/*
+				 * This talloc_memdup() is OK with the
+				 * +1 because *d has been set to '\0'
+				 * just above
+				 */
+				dn->components[dn->comp_num].value.data = \
+					(uint8_t *)talloc_memdup(dn->components, dt, l + 1);
+				dn->components[dn->comp_num].value.length = l;
+				if (dn->components[dn->comp_num].value.data == NULL) {
+					/* ouch ! */
+					goto failed;
+				}
+				talloc_set_name_const(dn->components[dn->comp_num].value.data,
+						      (const char *)dn->components[dn->comp_num].value.data);
+
+				dt = d;
+
+				dn->comp_num++;
+				if (dn->comp_num > 2) {
+					dn->components = talloc_realloc(dn,
+									dn->components,
+									struct ldb_dn_component,
+									dn->comp_num + 1);
+					if (dn->components == NULL) {
+						/* ouch ! */
+						goto failed;
+					}
+					/* make sure all components are zeroed, other functions depend on this */
+					memset(&dn->components[dn->comp_num], '\0', sizeof(struct ldb_dn_component));
+				}
+
+				continue;
+
+			case '+':
+			case '=':
+				/* to main compatibility with earlier
+				versions of ldb indexing, we have to
+				accept the base64 encoded binary index
+				values, which contain a '+' or '='
+				which should normally be escaped */
+				if (is_index) {
+					if (t != NULL) {
+						t = NULL;
+					}
+					*d++ = *p++;
+					l++;
+					break;
+				}
+
+				FALL_THROUGH;
+			case '\"':
+			case '<':
+			case '>':
+			case ';':
+				/* a string with not escaped specials is invalid (tested) */
+				if (!escape) {
+					ldb_dn_mark_invalid(dn);
+					goto failed;
+				}
+				escape = false;
+
+				*d++ = *p++;
+				l++;
+
+				if (t != NULL) {
+					t = NULL;
+				}
+				break;
+
+			case '\\':
+				if (!escape) {
+					escape = true;
+					p++;
+					continue;
+				}
+				escape = false;
+
+				*d++ = *p++;
+				l++;
+
+				if (t != NULL) {
+					t = NULL;
+				}
+				break;
+
+			default:
+				if (escape) {
+					if (isxdigit(p[0]) && isxdigit(p[1])) {
+						if (sscanf(p, "%02x", &x) != 1) {
+							/* invalid escaping sequence */
+							ldb_dn_mark_invalid(dn);
+							goto failed;
+						}
+						p += 2;
+						*d++ = (unsigned char)x;
+					} else {
+						*d++ = *p++;
+					}
+
+					escape = false;
+					l++;
+					if (t != NULL) {
+						t = NULL;
+					}
+					break;
+				}
+
+				if (*p == ' ') {
+					if (t == NULL) {
+						t = p;
+					}
+				} else {
+					if (t != NULL) {
+						t = NULL;
+					}
+				}
+
+				*d++ = *p++;
+				l++;
+
+				break;
+			}
+
+		}
+	}
+
+	if (in_attr || in_quote) {
+		/* invalid dn */
+		ldb_dn_mark_invalid(dn);
+		goto failed;
+	}
+
+	if (in_value) {
+		/* save last element */
+		if (t != NULL) {
+			/* trim back */
+			d -= (p - t);
+			l -= (p - t);
+		}
+
+		*d++ = '\0';
+		/*
+		 * This talloc_memdup() is OK with the
+		 * +1 because *d has been set to '\0'
+		 * just above.
+		 */
+		dn->components[dn->comp_num].value.length = l;
+		dn->components[dn->comp_num].value.data =
+			(uint8_t *)talloc_memdup(dn->components, dt, l + 1);
+		if (dn->components[dn->comp_num].value.data == NULL) {
+			/* ouch */
+			goto failed;
+		}
+		talloc_set_name_const(dn->components[dn->comp_num].value.data,
+			(const char *)dn->components[dn->comp_num].value.data);
+
+		dn->comp_num++;
+	}
+	talloc_free(data);
+	return true;
+
+failed:
+	LDB_FREE(dn->components); /* "data" is implicitly free'd */
+	dn->comp_num = 0;
+	LDB_FREE(dn->ext_components);
+	dn->ext_comp_num = 0;
+
+	return false;
+}
+
+bool ldb_dn_validate(struct ldb_dn *dn)
+{
+	return ldb_dn_explode(dn);
+}
+
+const char *ldb_dn_get_linearized(struct ldb_dn *dn)
+{
+	unsigned int i;
+	size_t len;
+	char *d, *n;
+
+	if ( ! dn || ( dn->invalid)) return NULL;
+
+	if (dn->linearized) return dn->linearized;
+
+	if ( ! dn->components) {
+		ldb_dn_mark_invalid(dn);
+		return NULL;
+	}
+
+	if (dn->comp_num == 0) {
+		dn->linearized = talloc_strdup(dn, "");
+		if ( ! dn->linearized) return NULL;
+		return dn->linearized;
+	}
+
+	/* calculate maximum possible length of DN */
+	for (len = 0, i = 0; i < dn->comp_num; i++) {
+		/* name len */
+		len += strlen(dn->components[i].name);
+		/* max escaped data len */
+		len += (dn->components[i].value.length * 3);
+		len += 2; /* '=' and ',' */
+	}
+	dn->linearized = talloc_array(dn, char, len);
+	if ( ! dn->linearized) return NULL;
+
+	d = dn->linearized;
+
+	for (i = 0; i < dn->comp_num; i++) {
+
+		/* copy the name */
+		n = dn->components[i].name;
+		while (*n) *d++ = *n++;
+
+		*d++ = '=';
+
+		/* and the value */
+		d += ldb_dn_escape_internal( d,
+				(char *)dn->components[i].value.data,
+				dn->components[i].value.length);
+		*d++ = ',';
+	}
+
+	*(--d) = '\0';
+
+	/* don't waste more memory than necessary */
+	dn->linearized = talloc_realloc(dn, dn->linearized,
+					char, (d - dn->linearized + 1));
+
+	return dn->linearized;
+}
+
+static int ldb_dn_extended_component_compare(const void *p1, const void *p2)
+{
+	const struct ldb_dn_ext_component *ec1 = (const struct ldb_dn_ext_component *)p1;
+	const struct ldb_dn_ext_component *ec2 = (const struct ldb_dn_ext_component *)p2;
+	return strcmp(ec1->name, ec2->name);
+}
+
+char *ldb_dn_get_extended_linearized(TALLOC_CTX *mem_ctx, struct ldb_dn *dn, int mode)
+{
+	const char *linearized = ldb_dn_get_linearized(dn);
+	char *p = NULL;
+	unsigned int i;
+
+	if (!linearized) {
+		return NULL;
+	}
+
+	if (!ldb_dn_has_extended(dn)) {
+		return talloc_strdup(mem_ctx, linearized);
+	}
+
+	if (!ldb_dn_validate(dn)) {
+		return NULL;
+	}
+
+	/* sort the extended components by name. The idea is to make
+	 * the resulting DNs consistent, plus to ensure that we put
+	 * 'DELETED' first, so it can be very quickly recognised
+	 */
+	TYPESAFE_QSORT(dn->ext_components, dn->ext_comp_num,
+		       ldb_dn_extended_component_compare);
+
+	for (i = 0; i < dn->ext_comp_num; i++) {
+		const struct ldb_dn_extended_syntax *ext_syntax;
+		const char *name = dn->ext_components[i].name;
+		struct ldb_val ec_val = dn->ext_components[i].value;
+		struct ldb_val val;
+		int ret;
+
+		ext_syntax = ldb_dn_extended_syntax_by_name(dn->ldb, name);
+		if (!ext_syntax) {
+			return NULL;
+		}
+
+		if (mode == 1) {
+			ret = ext_syntax->write_clear_fn(dn->ldb, mem_ctx,
+							&ec_val, &val);
+		} else if (mode == 0) {
+			ret = ext_syntax->write_hex_fn(dn->ldb, mem_ctx,
+							&ec_val, &val);
+		} else {
+			ret = -1;
+		}
+
+		if (ret != LDB_SUCCESS) {
+			return NULL;
+		}
+
+		if (i == 0) {
+			p = talloc_asprintf(mem_ctx, "<%s=%.*s>",
+					    name,
+					    (int)val.length,
+					    val.data);
+		} else {
+			p = talloc_asprintf_append_buffer(p, ";<%s=%.*s>",
+							  name,
+							  (int)val.length,
+							  val.data);
+		}
+
+		talloc_free(val.data);
+
+		if (!p) {
+			return NULL;
+		}
+	}
+
+	if (dn->ext_comp_num && *linearized) {
+		p = talloc_asprintf_append_buffer(p, ";%s", linearized);
+	}
+
+	if (!p) {
+		return NULL;
+	}
+
+	return p;
+}
+
+/*
+  filter out all but an acceptable list of extended DN components
+ */
+void ldb_dn_extended_filter(struct ldb_dn *dn, const char * const *accept_list)
+{
+	unsigned int i;
+	for (i=0; i<dn->ext_comp_num; i++) {
+		if (!ldb_attr_in_list(accept_list, dn->ext_components[i].name)) {
+			ARRAY_DEL_ELEMENT(
+				dn->ext_components, i, dn->ext_comp_num);
+			dn->ext_comp_num--;
+			i--;
+		}
+	}
+	LDB_FREE(dn->ext_linearized);
+}
+
+
+char *ldb_dn_alloc_linearized(TALLOC_CTX *mem_ctx, struct ldb_dn *dn)
+{
+	return talloc_strdup(mem_ctx, ldb_dn_get_linearized(dn));
+}
+
+/*
+  casefold a dn. We need to casefold the attribute names, and canonicalize
+  attribute values of case insensitive attributes.
+*/
+
+static bool ldb_dn_casefold_internal(struct ldb_dn *dn)
+{
+	unsigned int i;
+	int ret;
+
+	if ( ! dn || dn->invalid) return false;
+
+	if (dn->valid_case) return true;
+
+	if (( ! dn->components) && ( ! ldb_dn_explode(dn))) {
+		return false;
+	}
+
+	for (i = 0; i < dn->comp_num; i++) {
+		const struct ldb_schema_attribute *a;
+
+		dn->components[i].cf_name =
+			ldb_attr_casefold(dn->components,
+					  dn->components[i].name);
+		if (!dn->components[i].cf_name) {
+			goto failed;
+		}
+
+		a = ldb_schema_attribute_by_name(dn->ldb,
+						 dn->components[i].cf_name);
+
+		ret = a->syntax->canonicalise_fn(dn->ldb, dn->components,
+						 &(dn->components[i].value),
+						 &(dn->components[i].cf_value));
+		if (ret != 0) {
+			goto failed;
+		}
+	}
+
+	dn->valid_case = true;
+
+	return true;
+
+failed:
+	for (i = 0; i < dn->comp_num; i++) {
+		LDB_FREE(dn->components[i].cf_name);
+		LDB_FREE(dn->components[i].cf_value.data);
+	}
+	return false;
+}
+
+const char *ldb_dn_get_casefold(struct ldb_dn *dn)
+{
+	unsigned int i;
+	size_t len;
+	char *d, *n;
+
+	if (dn->casefold) return dn->casefold;
+
+	if (dn->special) {
+		dn->casefold = talloc_strdup(dn, dn->linearized);
+		if (!dn->casefold) return NULL;
+		dn->valid_case = true;
+		return dn->casefold;
+	}
+
+	if ( ! ldb_dn_casefold_internal(dn)) {
+		return NULL;
+	}
+
+	if (dn->comp_num == 0) {
+		dn->casefold = talloc_strdup(dn, "");
+		return dn->casefold;
+	}
+
+	/* calculate maximum possible length of DN */
+	for (len = 0, i = 0; i < dn->comp_num; i++) {
+		/* name len */
+		len += strlen(dn->components[i].cf_name);
+		/* max escaped data len */
+		len += (dn->components[i].cf_value.length * 3);
+		len += 2; /* '=' and ',' */
+	}
+	dn->casefold = talloc_array(dn, char, len);
+	if ( ! dn->casefold) return NULL;
+
+	d = dn->casefold;
+
+	for (i = 0; i < dn->comp_num; i++) {
+
+		/* copy the name */
+		n = dn->components[i].cf_name;
+		while (*n) *d++ = *n++;
+
+		*d++ = '=';
+
+		/* and the value */
+		d += ldb_dn_escape_internal( d,
+				(char *)dn->components[i].cf_value.data,
+				dn->components[i].cf_value.length);
+		*d++ = ',';
+	}
+	*(--d) = '\0';
+
+	/* don't waste more memory than necessary */
+	dn->casefold = talloc_realloc(dn, dn->casefold,
+				      char, strlen(dn->casefold) + 1);
+
+	return dn->casefold;
+}
+
+char *ldb_dn_alloc_casefold(TALLOC_CTX *mem_ctx, struct ldb_dn *dn)
+{
+	return talloc_strdup(mem_ctx, ldb_dn_get_casefold(dn));
+}
+
+/* Determine if dn is below base, in the ldap tree.  Used for
+ * evaluating a subtree search.
+ * 0 if they match, otherwise non-zero
+ */
+
+int ldb_dn_compare_base(struct ldb_dn *base, struct ldb_dn *dn)
+{
+	int ret;
+	unsigned int n_base, n_dn;
+
+	if ( ! base || base->invalid) return 1;
+	if ( ! dn || dn->invalid) return -1;
+
+	if (( ! base->valid_case) || ( ! dn->valid_case)) {
+		if (base->linearized && dn->linearized && dn->special == base->special) {
+			/* try with a normal compare first, if we are lucky
+			 * we will avoid exploding and casfolding */
+			int dif;
+			dif = strlen(dn->linearized) - strlen(base->linearized);
+			if (dif < 0) {
+				return dif;
+			}
+			if (strcmp(base->linearized,
+				   &dn->linearized[dif]) == 0) {
+				return 0;
+			}
+		}
+
+		if ( ! ldb_dn_casefold_internal(base)) {
+			return 1;
+		}
+
+		if ( ! ldb_dn_casefold_internal(dn)) {
+			return -1;
+		}
+
+	}
+
+	/* if base has more components,
+	 * they don't have the same base */
+	if (base->comp_num > dn->comp_num) {
+		return (dn->comp_num - base->comp_num);
+	}
+
+	if ((dn->comp_num == 0) || (base->comp_num == 0)) {
+		if (dn->special && base->special) {
+			return strcmp(base->linearized, dn->linearized);
+		} else if (dn->special) {
+			return -1;
+		} else if (base->special) {
+			return 1;
+		} else {
+			return 0;
+		}
+	}
+
+	n_base = base->comp_num - 1;
+	n_dn = dn->comp_num - 1;
+
+	while (n_base != (unsigned int) -1) {
+		char *b_name = base->components[n_base].cf_name;
+		char *dn_name = dn->components[n_dn].cf_name;
+
+		char *b_vdata = (char *)base->components[n_base].cf_value.data;
+		char *dn_vdata = (char *)dn->components[n_dn].cf_value.data;
+
+		size_t b_vlen = base->components[n_base].cf_value.length;
+		size_t dn_vlen = dn->components[n_dn].cf_value.length;
+
+		/* compare attr names */
+		ret = strcmp(b_name, dn_name);
+		if (ret != 0) return ret;
+
+		/* compare attr.cf_value. */
+		if (b_vlen != dn_vlen) {
+			return b_vlen - dn_vlen;
+		}
+		ret = strncmp(b_vdata, dn_vdata, b_vlen);
+		if (ret != 0) return ret;
+
+		n_base--;
+		n_dn--;
+	}
+
+	return 0;
+}
+
+/* compare DNs using casefolding compare functions.
+
+   If they match, then return 0
+ */
+
+int ldb_dn_compare(struct ldb_dn *dn0, struct ldb_dn *dn1)
+{
+	unsigned int i;
+	int ret;
+
+	if (( ! dn0) || dn0->invalid || ! dn1 || dn1->invalid) {
+		return -1;
+	}
+
+	if (( ! dn0->valid_case) || ( ! dn1->valid_case)) {
+		if (dn0->linearized && dn1->linearized) {
+			/* try with a normal compare first, if we are lucky
+			 * we will avoid exploding and casfolding */
+			if (strcmp(dn0->linearized, dn1->linearized) == 0) {
+				return 0;
+			}
+		}
+
+		if ( ! ldb_dn_casefold_internal(dn0)) {
+			return 1;
+		}
+
+		if ( ! ldb_dn_casefold_internal(dn1)) {
+			return -1;
+		}
+
+	}
+
+	if (dn0->comp_num != dn1->comp_num) {
+		return (dn1->comp_num - dn0->comp_num);
+	}
+
+	if (dn0->comp_num == 0) {
+		if (dn0->special && dn1->special) {
+			return strcmp(dn0->linearized, dn1->linearized);
+		} else if (dn0->special) {
+			return 1;
+		} else if (dn1->special) {
+			return -1;
+		} else {
+			return 0;
+		}
+	}
+
+	for (i = 0; i < dn0->comp_num; i++) {
+		char *dn0_name = dn0->components[i].cf_name;
+		char *dn1_name = dn1->components[i].cf_name;
+
+		char *dn0_vdata = (char *)dn0->components[i].cf_value.data;
+		char *dn1_vdata = (char *)dn1->components[i].cf_value.data;
+
+		size_t dn0_vlen = dn0->components[i].cf_value.length;
+		size_t dn1_vlen = dn1->components[i].cf_value.length;
+
+		/* compare attr names */
+		ret = strcmp(dn0_name, dn1_name);
+		if (ret != 0) {
+			return ret;
+		}
+
+		/* compare attr.cf_value. */
+		if (dn0_vlen != dn1_vlen) {
+			return dn0_vlen - dn1_vlen;
+		}
+		ret = strncmp(dn0_vdata, dn1_vdata, dn0_vlen);
+		if (ret != 0) {
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static struct ldb_dn_component ldb_dn_copy_component(
+						TALLOC_CTX *mem_ctx,
+						struct ldb_dn_component *src)
+{
+	struct ldb_dn_component dst;
+
+	memset(&dst, 0, sizeof(dst));
+
+	if (src == NULL) {
+		return dst;
+	}
+
+	dst.value = ldb_val_dup(mem_ctx, &(src->value));
+	if (dst.value.data == NULL) {
+		return dst;
+	}
+
+	dst.name = talloc_strdup(mem_ctx, src->name);
+	if (dst.name == NULL) {
+		LDB_FREE(dst.value.data);
+		return dst;
+	}
+
+	if (src->cf_value.data) {
+		dst.cf_value = ldb_val_dup(mem_ctx, &(src->cf_value));
+		if (dst.cf_value.data == NULL) {
+			LDB_FREE(dst.value.data);
+			LDB_FREE(dst.name);
+			return dst;
+		}
+
+		dst.cf_name = talloc_strdup(mem_ctx, src->cf_name);
+		if (dst.cf_name == NULL) {
+			LDB_FREE(dst.cf_name);
+			LDB_FREE(dst.value.data);
+			LDB_FREE(dst.name);
+			return dst;
+		}
+	} else {
+		dst.cf_value.data = NULL;
+		dst.cf_name = NULL;
+	}
+
+	return dst;
+}
+
+static struct ldb_dn_ext_component ldb_dn_ext_copy_component(
+						TALLOC_CTX *mem_ctx,
+						struct ldb_dn_ext_component *src)
+{
+	struct ldb_dn_ext_component dst;
+
+	memset(&dst, 0, sizeof(dst));
+
+	if (src == NULL) {
+		return dst;
+	}
+
+	dst.value = ldb_val_dup(mem_ctx, &(src->value));
+	if (dst.value.data == NULL) {
+		return dst;
+	}
+
+	dst.name = talloc_strdup(mem_ctx, src->name);
+	if (dst.name == NULL) {
+		LDB_FREE(dst.value.data);
+		return dst;
+	}
+
+	return dst;
+}
+
+struct ldb_dn *ldb_dn_copy(TALLOC_CTX *mem_ctx, struct ldb_dn *dn)
+{
+	struct ldb_dn *new_dn;
+
+	if (!dn || dn->invalid) {
+		return NULL;
+	}
+
+	new_dn = talloc_zero(mem_ctx, struct ldb_dn);
+	if ( !new_dn) {
+		return NULL;
+	}
+
+	*new_dn = *dn;
+
+	if (dn->components) {
+		unsigned int i;
+
+		new_dn->components =
+			talloc_zero_array(new_dn,
+					  struct ldb_dn_component,
+					  dn->comp_num);
+		if ( ! new_dn->components) {
+			talloc_free(new_dn);
+			return NULL;
+		}
+
+		for (i = 0; i < dn->comp_num; i++) {
+			new_dn->components[i] =
+				ldb_dn_copy_component(new_dn->components,
+						      &dn->components[i]);
+			if ( ! new_dn->components[i].value.data) {
+				talloc_free(new_dn);
+				return NULL;
+			}
+		}
+	}
+
+	if (dn->ext_components) {
+		unsigned int i;
+
+		new_dn->ext_components =
+			talloc_zero_array(new_dn,
+					  struct ldb_dn_ext_component,
+					  dn->ext_comp_num);
+		if ( ! new_dn->ext_components) {
+			talloc_free(new_dn);
+			return NULL;
+		}
+
+		for (i = 0; i < dn->ext_comp_num; i++) {
+			new_dn->ext_components[i] =
+				 ldb_dn_ext_copy_component(
+						new_dn->ext_components,
+						&dn->ext_components[i]);
+			if ( ! new_dn->ext_components[i].value.data) {
+				talloc_free(new_dn);
+				return NULL;
+			}
+		}
+	}
+
+	if (dn->casefold) {
+		new_dn->casefold = talloc_strdup(new_dn, dn->casefold);
+		if ( ! new_dn->casefold) {
+			talloc_free(new_dn);
+			return NULL;
+		}
+	}
+
+	if (dn->linearized) {
+		new_dn->linearized = talloc_strdup(new_dn, dn->linearized);
+		if ( ! new_dn->linearized) {
+			talloc_free(new_dn);
+			return NULL;
+		}
+	}
+
+	if (dn->ext_linearized) {
+		new_dn->ext_linearized = talloc_strdup(new_dn,
+							dn->ext_linearized);
+		if ( ! new_dn->ext_linearized) {
+			talloc_free(new_dn);
+			return NULL;
+		}
+	}
+
+	return new_dn;
+}
+
+/* modify the given dn by adding a base.
+ *
+ * return true if successful and false if not
+ * if false is returned the dn may be marked invalid
+ */
+bool ldb_dn_add_base(struct ldb_dn *dn, struct ldb_dn *base)
+{
+	const char *s;
+	char *t;
+
+	if ( !base || base->invalid || !dn || dn->invalid) {
+		return false;
+	}
+
+	if (dn == base) {
+		return false; /* or we will visit infinity */
+	}
+
+	if (dn->components) {
+		unsigned int i;
+
+		if ( ! ldb_dn_validate(base)) {
+			return false;
+		}
+
+		s = NULL;
+		if (dn->valid_case) {
+			if ( ! (s = ldb_dn_get_casefold(base))) {
+				return false;
+			}
+		}
+
+		dn->components = talloc_realloc(dn,
+						dn->components,
+						struct ldb_dn_component,
+						dn->comp_num + base->comp_num);
+		if ( ! dn->components) {
+			ldb_dn_mark_invalid(dn);
+			return false;
+		}
+
+		for (i = 0; i < base->comp_num; dn->comp_num++, i++) {
+			dn->components[dn->comp_num] =
+				ldb_dn_copy_component(dn->components,
+							&base->components[i]);
+			if (dn->components[dn->comp_num].value.data == NULL) {
+				ldb_dn_mark_invalid(dn);
+				return false;
+			}
+		}
+
+		if (dn->casefold && s) {
+			if (*dn->casefold) {
+				t = talloc_asprintf(dn, "%s,%s",
+						    dn->casefold, s);
+			} else {
+				t = talloc_strdup(dn, s);
+			}
+			LDB_FREE(dn->casefold);
+			dn->casefold = t;
+		}
+	}
+
+	if (dn->linearized) {
+
+		s = ldb_dn_get_linearized(base);
+		if ( ! s) {
+			return false;
+		}
+
+		if (*dn->linearized) {
+			t = talloc_asprintf(dn, "%s,%s",
+					    dn->linearized, s);
+		} else {
+			t = talloc_strdup(dn, s);
+		}
+		if ( ! t) {
+			ldb_dn_mark_invalid(dn);
+			return false;
+		}
+		LDB_FREE(dn->linearized);
+		dn->linearized = t;
+	}
+
+	/* Wipe the ext_linearized DN,
+	 * the GUID and SID are almost certainly no longer valid */
+	LDB_FREE(dn->ext_linearized);
+	LDB_FREE(dn->ext_components);
+	dn->ext_comp_num = 0;
+
+	return true;
+}
+
+/* modify the given dn by adding a base.
+ *
+ * return true if successful and false if not
+ * if false is returned the dn may be marked invalid
+ */
+bool ldb_dn_add_base_fmt(struct ldb_dn *dn, const char *base_fmt, ...)
+{
+	struct ldb_dn *base;
+	char *base_str;
+	va_list ap;
+	bool ret;
+
+	if ( !dn || dn->invalid) {
+		return false;
+	}
+
+	va_start(ap, base_fmt);
+	base_str = talloc_vasprintf(dn, base_fmt, ap);
+	va_end(ap);
+
+	if (base_str == NULL) {
+		return false;
+	}
+
+	base = ldb_dn_new(base_str, dn->ldb, base_str);
+
+	ret = ldb_dn_add_base(dn, base);
+
+	talloc_free(base_str);
+
+	return ret;
+}
+
+/* modify the given dn by adding children elements.
+ *
+ * return true if successful and false if not
+ * if false is returned the dn may be marked invalid
+ */
+bool ldb_dn_add_child(struct ldb_dn *dn, struct ldb_dn *child)
+{
+	const char *s;
+	char *t;
+
+	if ( !child || child->invalid || !dn || dn->invalid) {
+		return false;
+	}
+
+	if (dn->components) {
+		unsigned int n;
+		unsigned int i, j;
+
+		if (dn->comp_num == 0) {
+			return false;
+		}
+
+		if ( ! ldb_dn_validate(child)) {
+			return false;
+		}
+
+		s = NULL;
+		if (dn->valid_case) {
+			if ( ! (s = ldb_dn_get_casefold(child))) {
+				return false;
+			}
+		}
+
+		n = dn->comp_num + child->comp_num;
+
+		dn->components = talloc_realloc(dn,
+						dn->components,
+						struct ldb_dn_component,
+						n);
+		if ( ! dn->components) {
+			ldb_dn_mark_invalid(dn);
+			return false;
+		}
+
+		for (i = dn->comp_num - 1, j = n - 1; i != (unsigned int) -1;
+		     i--, j--) {
+			dn->components[j] = dn->components[i];
+		}
+
+		for (i = 0; i < child->comp_num; i++) {
+			dn->components[i] =
+				ldb_dn_copy_component(dn->components,
+							&child->components[i]);
+			if (dn->components[i].value.data == NULL) {
+				ldb_dn_mark_invalid(dn);
+				return false;
+			}
+		}
+
+		dn->comp_num = n;
+
+		if (dn->casefold && s) {
+			t = talloc_asprintf(dn, "%s,%s", s, dn->casefold);
+			LDB_FREE(dn->casefold);
+			dn->casefold = t;
+		}
+	}
+
+	if (dn->linearized) {
+		if (dn->linearized[0] == '\0') {
+			return false;
+		}
+
+		s = ldb_dn_get_linearized(child);
+		if ( ! s) {
+			return false;
+		}
+
+		t = talloc_asprintf(dn, "%s,%s", s, dn->linearized);
+		if ( ! t) {
+			ldb_dn_mark_invalid(dn);
+			return false;
+		}
+		LDB_FREE(dn->linearized);
+		dn->linearized = t;
+	}
+
+	/* Wipe the ext_linearized DN,
+	 * the GUID and SID are almost certainly no longer valid */
+	LDB_FREE(dn->ext_linearized);
+	LDB_FREE(dn->ext_components);
+	dn->ext_comp_num = 0;
+
+	return true;
+}
+
+/* modify the given dn by adding children elements.
+ *
+ * return true if successful and false if not
+ * if false is returned the dn may be marked invalid
+ */
+bool ldb_dn_add_child_fmt(struct ldb_dn *dn, const char *child_fmt, ...)
+{
+	struct ldb_dn *child;
+	char *child_str;
+	va_list ap;
+	bool ret;
+
+	if ( !dn || dn->invalid) {
+		return false;
+	}
+
+	va_start(ap, child_fmt);
+	child_str = talloc_vasprintf(dn, child_fmt, ap);
+	va_end(ap);
+
+	if (child_str == NULL) {
+		return false;
+	}
+
+	child = ldb_dn_new(child_str, dn->ldb, child_str);
+
+	ret = ldb_dn_add_child(dn, child);
+
+	talloc_free(child_str);
+
+	return ret;
+}
+
+/* modify the given dn by adding a single child element.
+ *
+ * return true if successful and false if not
+ * if false is returned the dn may be marked invalid
+ */
+bool ldb_dn_add_child_val(struct ldb_dn *dn,
+			  const char *rdn,
+			  struct ldb_val value)
+{
+	bool ret;
+	int ldb_ret;
+	struct ldb_dn *child = NULL;
+
+	if ( !dn || dn->invalid) {
+		return false;
+	}
+
+	child = ldb_dn_new(dn, dn->ldb, "X=Y");
+	ret = ldb_dn_add_child(dn, child);
+
+	if (ret == false) {
+		return false;
+	}
+
+	ldb_ret = ldb_dn_set_component(dn,
+				       0,
+				       rdn,
+				       value);
+	if (ldb_ret != LDB_SUCCESS) {
+		return false;
+	}
+
+	return true;
+}
+
+bool ldb_dn_remove_base_components(struct ldb_dn *dn, unsigned int num)
+{
+	unsigned int i;
+
+	if ( ! ldb_dn_validate(dn)) {
+		return false;
+	}
+
+	if (dn->comp_num < num) {
+		return false;
+	}
+
+	/* free components */
+	for (i = dn->comp_num - num; i < dn->comp_num; i++) {
+		LDB_FREE(dn->components[i].name);
+		LDB_FREE(dn->components[i].value.data);
+		LDB_FREE(dn->components[i].cf_name);
+		LDB_FREE(dn->components[i].cf_value.data);
+	}
+
+	dn->comp_num -= num;
+
+	if (dn->valid_case) {
+		for (i = 0; i < dn->comp_num; i++) {
+			LDB_FREE(dn->components[i].cf_name);
+			LDB_FREE(dn->components[i].cf_value.data);
+		}
+		dn->valid_case = false;
+	}
+
+	LDB_FREE(dn->casefold);
+	LDB_FREE(dn->linearized);
+
+	/* Wipe the ext_linearized DN,
+	 * the GUID and SID are almost certainly no longer valid */
+	LDB_FREE(dn->ext_linearized);
+	LDB_FREE(dn->ext_components);
+	dn->ext_comp_num = 0;
+
+	return true;
+}
+
+bool ldb_dn_remove_child_components(struct ldb_dn *dn, unsigned int num)
+{
+	unsigned int i, j;
+
+	if ( ! ldb_dn_validate(dn)) {
+		return false;
+	}
+
+	if (dn->comp_num < num) {
+		return false;
+	}
+
+	for (i = 0, j = num; j < dn->comp_num; i++, j++) {
+		if (i < num) {
+			LDB_FREE(dn->components[i].name);
+			LDB_FREE(dn->components[i].value.data);
+			LDB_FREE(dn->components[i].cf_name);
+			LDB_FREE(dn->components[i].cf_value.data);
+		}
+		dn->components[i] = dn->components[j];
+	}
+
+	dn->comp_num -= num;
+
+	if (dn->valid_case) {
+		for (i = 0; i < dn->comp_num; i++) {
+			LDB_FREE(dn->components[i].cf_name);
+			LDB_FREE(dn->components[i].cf_value.data);
+		}
+		dn->valid_case = false;
+	}
+
+	LDB_FREE(dn->casefold);
+	LDB_FREE(dn->linearized);
+
+	/* Wipe the ext_linearized DN,
+	 * the GUID and SID are almost certainly no longer valid */
+	LDB_FREE(dn->ext_linearized);
+	LDB_FREE(dn->ext_components);
+	dn->ext_comp_num = 0;
+
+	return true;
+}
+
+
+/* replace the components of a DN with those from another DN, without
+ * touching the extended components
+ *
+ * return true if successful and false if not
+ * if false is returned the dn may be marked invalid
+ */
+bool ldb_dn_replace_components(struct ldb_dn *dn, struct ldb_dn *new_dn)
+{
+	unsigned int i;
+
+	if ( ! ldb_dn_validate(dn) || ! ldb_dn_validate(new_dn)) {
+		return false;
+	}
+
+	/* free components */
+	for (i = 0; i < dn->comp_num; i++) {
+		LDB_FREE(dn->components[i].name);
+		LDB_FREE(dn->components[i].value.data);
+		LDB_FREE(dn->components[i].cf_name);
+		LDB_FREE(dn->components[i].cf_value.data);
+	}
+
+	dn->components = talloc_realloc(dn,
+					dn->components,
+					struct ldb_dn_component,
+					new_dn->comp_num);
+	if (dn->components == NULL) {
+		ldb_dn_mark_invalid(dn);
+		return false;
+	}
+
+	dn->comp_num = new_dn->comp_num;
+	dn->valid_case = new_dn->valid_case;
+
+	for (i = 0; i < dn->comp_num; i++) {
+		dn->components[i] = ldb_dn_copy_component(dn->components, &new_dn->components[i]);
+		if (dn->components[i].name == NULL) {
+			ldb_dn_mark_invalid(dn);
+			return false;
+		}
+	}
+	if (new_dn->linearized == NULL) {
+		dn->linearized = NULL;
+	} else {
+		dn->linearized = talloc_strdup(dn, new_dn->linearized);
+		if (dn->linearized == NULL) {
+			ldb_dn_mark_invalid(dn);
+			return false;
+		}
+	}
+
+	return true;
+}
+
+
+struct ldb_dn *ldb_dn_get_parent(TALLOC_CTX *mem_ctx, struct ldb_dn *dn)
+{
+	struct ldb_dn *new_dn;
+
+	new_dn = ldb_dn_copy(mem_ctx, dn);
+	if ( !new_dn ) {
+		return NULL;
+	}
+
+	if ( ! ldb_dn_remove_child_components(new_dn, 1)) {
+		talloc_free(new_dn);
+		return NULL;
+	}
+
+	return new_dn;
+}
+
+/* Create a 'canonical name' string from a DN:
+
+   ie dc=samba,dc=org -> samba.org/
+      uid=administrator,ou=users,dc=samba,dc=org = samba.org/users/administrator
+
+   There are two formats,
+   the EX format has the last '/' replaced with a newline (\n).
+
+*/
+static char *ldb_dn_canonical(TALLOC_CTX *mem_ctx, struct ldb_dn *dn, int ex_format) {
+	unsigned int i;
+	TALLOC_CTX *tmpctx;
+	char *cracked = NULL;
+	const char *format = (ex_format ? "\n" : "/" );
+
+	if ( ! ldb_dn_validate(dn)) {
+		return NULL;
+	}
+
+	tmpctx = talloc_new(mem_ctx);
+
+	/* Walk backwards down the DN, grabbing 'dc' components at first */
+	for (i = dn->comp_num - 1; i != (unsigned int) -1; i--) {
+		if (ldb_attr_cmp(dn->components[i].name, "dc") != 0) {
+			break;
+		}
+		if (cracked) {
+			cracked = talloc_asprintf(tmpctx, "%s.%s",
+						  ldb_dn_escape_value(tmpctx,
+							dn->components[i].value),
+						  cracked);
+		} else {
+			cracked = ldb_dn_escape_value(tmpctx,
+							dn->components[i].value);
+		}
+		if (!cracked) {
+			goto done;
+		}
+	}
+
+	/* Only domain components?  Finish here */
+	if (i == (unsigned int) -1) {
+		cracked = talloc_strdup_append_buffer(cracked, format);
+		talloc_steal(mem_ctx, cracked);
+		goto done;
+	}
+
+	/* Now walk backwards appending remaining components */
+	for (; i > 0; i--) {
+		cracked = talloc_asprintf_append_buffer(cracked, "/%s",
+							ldb_dn_escape_value(tmpctx,
+							dn->components[i].value));
+		if (!cracked) {
+			goto done;
+		}
+	}
+
+	/* Last one, possibly a newline for the 'ex' format */
+	cracked = talloc_asprintf_append_buffer(cracked, "%s%s", format,
+						ldb_dn_escape_value(tmpctx,
+							dn->components[i].value));
+
+	talloc_steal(mem_ctx, cracked);
+done:
+	talloc_free(tmpctx);
+	return cracked;
+}
+
+/* Wrapper functions for the above, for the two different string formats */
+char *ldb_dn_canonical_string(TALLOC_CTX *mem_ctx, struct ldb_dn *dn) {
+	return ldb_dn_canonical(mem_ctx, dn, 0);
+
+}
+
+char *ldb_dn_canonical_ex_string(TALLOC_CTX *mem_ctx, struct ldb_dn *dn) {
+	return ldb_dn_canonical(mem_ctx, dn, 1);
+}
+
+int ldb_dn_get_comp_num(struct ldb_dn *dn)
+{
+	if ( ! ldb_dn_validate(dn)) {
+		return -1;
+	}
+	return dn->comp_num;
+}
+
+int ldb_dn_get_extended_comp_num(struct ldb_dn *dn)
+{
+	if ( ! ldb_dn_validate(dn)) {
+		return -1;
+	}
+	return dn->ext_comp_num;
+}
+
+const char *ldb_dn_get_component_name(struct ldb_dn *dn, unsigned int num)
+{
+	if ( ! ldb_dn_validate(dn)) {
+		return NULL;
+	}
+	if (num >= dn->comp_num) return NULL;
+	return dn->components[num].name;
+}
+
+const struct ldb_val *ldb_dn_get_component_val(struct ldb_dn *dn,
+						unsigned int num)
+{
+	if ( ! ldb_dn_validate(dn)) {
+		return NULL;
+	}
+	if (num >= dn->comp_num) return NULL;
+	return &dn->components[num].value;
+}
+
+const char *ldb_dn_get_rdn_name(struct ldb_dn *dn)
+{
+	if ( ! ldb_dn_validate(dn)) {
+		return NULL;
+	}
+	if (dn->comp_num == 0) return NULL;
+	return dn->components[0].name;
+}
+
+const struct ldb_val *ldb_dn_get_rdn_val(struct ldb_dn *dn)
+{
+	if ( ! ldb_dn_validate(dn)) {
+		return NULL;
+	}
+	if (dn->comp_num == 0) return NULL;
+	return &dn->components[0].value;
+}
+
+int ldb_dn_set_component(struct ldb_dn *dn, int num,
+			 const char *name, const struct ldb_val val)
+{
+	char *n;
+	struct ldb_val v;
+
+	if ( ! ldb_dn_validate(dn)) {
+		return LDB_ERR_OTHER;
+	}
+
+	if (num < 0) {
+		return LDB_ERR_OTHER;
+	}
+
+	if ((unsigned)num >= dn->comp_num) {
+		return LDB_ERR_OTHER;
+	}
+
+	if (val.length > val.length + 1) {
+		return LDB_ERR_OTHER;
+	}
+
+	n = talloc_strdup(dn, name);
+	if ( ! n) {
+		return LDB_ERR_OTHER;
+	}
+
+	v.length = val.length;
+
+	/*
+	 * This is like talloc_memdup(dn, v.data, v.length + 1), but
+	 * avoids the over-read
+	 */
+	v.data = (uint8_t *)talloc_size(dn, v.length+1);
+	if ( ! v.data) {
+		talloc_free(n);
+		return LDB_ERR_OTHER;
+	}
+	memcpy(v.data, val.data, val.length);
+
+	/*
+	 * Enforce NUL termination outside the stated length, as is
+	 * traditional in LDB
+	 */
+	v.data[v.length] = '\0';
+
+	talloc_free(dn->components[num].name);
+	talloc_free(dn->components[num].value.data);
+	dn->components[num].name = n;
+	dn->components[num].value = v;
+
+	if (dn->valid_case) {
+		unsigned int i;
+		for (i = 0; i < dn->comp_num; i++) {
+			LDB_FREE(dn->components[i].cf_name);
+			LDB_FREE(dn->components[i].cf_value.data);
+		}
+		dn->valid_case = false;
+	}
+	LDB_FREE(dn->casefold);
+	LDB_FREE(dn->linearized);
+
+	/* Wipe the ext_linearized DN,
+	 * the GUID and SID are almost certainly no longer valid */
+	LDB_FREE(dn->ext_linearized);
+	LDB_FREE(dn->ext_components);
+	dn->ext_comp_num = 0;
+
+	return LDB_SUCCESS;
+}
+
+const struct ldb_val *ldb_dn_get_extended_component(struct ldb_dn *dn,
+						    const char *name)
+{
+	unsigned int i;
+	if ( ! ldb_dn_validate(dn)) {
+		return NULL;
+	}
+	for (i=0; i < dn->ext_comp_num; i++) {
+		if (ldb_attr_cmp(dn->ext_components[i].name, name) == 0) {
+			return &dn->ext_components[i].value;
+		}
+	}
+	return NULL;
+}
+
+int ldb_dn_set_extended_component(struct ldb_dn *dn,
+				  const char *name, const struct ldb_val *val)
+{
+	struct ldb_dn_ext_component *p;
+	unsigned int i;
+	struct ldb_val v2;
+	const struct ldb_dn_extended_syntax *ext_syntax;
+	
+	if ( ! ldb_dn_validate(dn)) {
+		return LDB_ERR_OTHER;
+	}
+
+	ext_syntax = ldb_dn_extended_syntax_by_name(dn->ldb, name);
+	if (ext_syntax == NULL) {
+		/* We don't know how to handle this type of thing */
+		return LDB_ERR_INVALID_DN_SYNTAX;
+	}
+
+	for (i=0; i < dn->ext_comp_num; i++) {
+		if (ldb_attr_cmp(dn->ext_components[i].name, name) == 0) {
+			if (val) {
+				dn->ext_components[i].value =
+					ldb_val_dup(dn->ext_components, val);
+
+				dn->ext_components[i].name = ext_syntax->name;
+				if (!dn->ext_components[i].value.data) {
+					ldb_dn_mark_invalid(dn);
+					return LDB_ERR_OPERATIONS_ERROR;
+				}
+			} else {
+				ARRAY_DEL_ELEMENT(
+					dn->ext_components,
+					i,
+					dn->ext_comp_num);
+				dn->ext_comp_num--;
+
+				dn->ext_components = talloc_realloc(dn,
+						   dn->ext_components,
+						   struct ldb_dn_ext_component,
+						   dn->ext_comp_num);
+				if (!dn->ext_components) {
+					ldb_dn_mark_invalid(dn);
+					return LDB_ERR_OPERATIONS_ERROR;
+				}
+			}
+			LDB_FREE(dn->ext_linearized);
+
+			return LDB_SUCCESS;
+		}
+	}
+
+	if (val == NULL) {
+		/* removing a value that doesn't exist is not an error */
+		return LDB_SUCCESS;
+	}
+
+	v2 = *val;
+
+	p = dn->ext_components
+		= talloc_realloc(dn,
+				 dn->ext_components,
+				 struct ldb_dn_ext_component,
+				 dn->ext_comp_num + 1);
+	if (!dn->ext_components) {
+		ldb_dn_mark_invalid(dn);
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	p[dn->ext_comp_num].value = ldb_val_dup(dn->ext_components, &v2);
+	p[dn->ext_comp_num].name = talloc_strdup(p, name);
+
+	if (!dn->ext_components[i].name || !dn->ext_components[i].value.data) {
+		ldb_dn_mark_invalid(dn);
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	dn->ext_components = p;
+	dn->ext_comp_num++;
+
+	LDB_FREE(dn->ext_linearized);
+
+	return LDB_SUCCESS;
+}
+
+void ldb_dn_remove_extended_components(struct ldb_dn *dn)
+{
+	LDB_FREE(dn->ext_linearized);
+	LDB_FREE(dn->ext_components);
+	dn->ext_comp_num = 0;
+}
+
+bool ldb_dn_is_valid(struct ldb_dn *dn)
+{
+	if ( ! dn) return false;
+	return ! dn->invalid;
+}
+
+bool ldb_dn_is_special(struct ldb_dn *dn)
+{
+	if ( ! dn || dn->invalid) return false;
+	return dn->special;
+}
+
+bool ldb_dn_has_extended(struct ldb_dn *dn)
+{
+	if ( ! dn || dn->invalid) return false;
+	if (dn->ext_linearized && (dn->ext_linearized[0] == '<')) return true;
+	return dn->ext_comp_num != 0;
+}
+
+bool ldb_dn_check_special(struct ldb_dn *dn, const char *check)
+{
+	if ( ! dn || dn->invalid) return false;
+	return ! strcmp(dn->linearized, check);
+}
+
+bool ldb_dn_is_null(struct ldb_dn *dn)
+{
+	if ( ! dn || dn->invalid) return false;
+	if (ldb_dn_has_extended(dn)) return false;
+	if (dn->linearized && (dn->linearized[0] == '\0')) return true;
+	return false;
+}
+
+/*
+  this updates dn->components, taking the components from ref_dn.
+  This is used by code that wants to update the DN path of a DN
+  while not impacting on the extended DN components
+ */
+int ldb_dn_update_components(struct ldb_dn *dn, const struct ldb_dn *ref_dn)
+{
+	dn->components = talloc_realloc(dn, dn->components,
+					struct ldb_dn_component, ref_dn->comp_num);
+	if (!dn->components) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	memcpy(dn->components, ref_dn->components,
+	       sizeof(struct ldb_dn_component)*ref_dn->comp_num);
+	dn->comp_num = ref_dn->comp_num;
+
+	LDB_FREE(dn->casefold);
+	LDB_FREE(dn->linearized);
+	LDB_FREE(dn->ext_linearized);
+
+	return LDB_SUCCESS;
+}
+
+/*
+  minimise a DN. The caller must pass in a validated DN.
+
+  If the DN has an extended component then only the first extended
+  component is kept, the DN string is stripped.
+
+  The existing dn is modified
+ */
+bool ldb_dn_minimise(struct ldb_dn *dn)
+{
+	unsigned int i;
+
+	if (!ldb_dn_validate(dn)) {
+		return false;
+	}
+	if (dn->ext_comp_num == 0) {
+		return true;
+	}
+
+	/* free components */
+	for (i = 0; i < dn->comp_num; i++) {
+		LDB_FREE(dn->components[i].name);
+		LDB_FREE(dn->components[i].value.data);
+		LDB_FREE(dn->components[i].cf_name);
+		LDB_FREE(dn->components[i].cf_value.data);
+	}
+	dn->comp_num = 0;
+	dn->valid_case = false;
+
+	LDB_FREE(dn->casefold);
+	LDB_FREE(dn->linearized);
+
+	/* note that we don't free dn->components as this there are
+	 * several places in ldb_dn.c that rely on it being non-NULL
+	 * for an exploded DN
+	 */
+
+	for (i = 1; i < dn->ext_comp_num; i++) {
+		LDB_FREE(dn->ext_components[i].value.data);
+	}
+	dn->ext_comp_num = 1;
+
+	dn->ext_components = talloc_realloc(dn, dn->ext_components, struct ldb_dn_ext_component, 1);
+	if (dn->ext_components == NULL) {
+		ldb_dn_mark_invalid(dn);
+		return false;
+	}
+
+	LDB_FREE(dn->ext_linearized);
+
+	return true;
+}
+
+struct ldb_context *ldb_dn_get_ldb_context(struct ldb_dn *dn)
+{
+	return dn->ldb;
+}
diff --git a/lib/ldb/common/ldb_ldif.c b/lib/ldb/common/ldb_ldif.c
new file mode 100644
index 0000000..6f7589f
--- /dev/null
+++ b/lib/ldb/common/ldb_ldif.c
@@ -0,0 +1,1108 @@
+/*
+   ldb database library
+
+   Copyright (C) Andrew Tridgell  2004
+
+     ** NOTE! The following LGPL license applies to the ldb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+/*
+ *  Name: ldb
+ *
+ *  Component: ldif routines
+ *
+ *  Description: ldif pack/unpack routines
+ *
+ *  Author: Andrew Tridgell
+ */
+
+/*
+  see RFC2849 for the LDIF format definition
+*/
+
+#include "ldb_private.h"
+#include "system/locale.h"
+
+/*
+
+*/
+static int ldb_read_data_file(TALLOC_CTX *mem_ctx, struct ldb_val *value)
+{
+	struct stat statbuf;
+	char *buf;
+	int count, size, bytes;
+	int ret;
+	int f;
+	const char *fname = (const char *)value->data;
+
+	if (strncmp(fname, "file://", 7) != 0) {
+		return LDB_ERR_INVALID_ATTRIBUTE_SYNTAX;
+	}
+	fname += 7;
+
+	f = open(fname, O_RDONLY);
+	if (f == -1) {
+		return -1;
+	}
+
+	if (fstat(f, &statbuf) != 0) {
+		ret = -1;
+		goto done;
+	}
+
+	if (statbuf.st_size == 0) {
+		ret = -1;
+		goto done;
+	}
+
+	value->data = (uint8_t *)talloc_size(mem_ctx, statbuf.st_size + 1);
+	if (value->data == NULL) {
+		ret = -1;
+		goto done;
+	}
+	value->data[statbuf.st_size] = 0;
+
+	count = 0;
+	size = statbuf.st_size;
+	buf = (char *)value->data;
+	while (count < statbuf.st_size) {
+		bytes = read(f, buf, size);
+		if (bytes == -1) {
+			talloc_free(value->data);
+			ret = -1;
+			goto done;
+		}
+		count += bytes;
+		buf += bytes;
+		size -= bytes;
+	}
+
+	value->length = statbuf.st_size;
+	ret = statbuf.st_size;
+
+done:
+	close(f);
+	return ret;
+}
+
+/*
+  this base64 decoder was taken from jitterbug (written by tridge).
+  we might need to replace it with a new version
+*/
+int ldb_base64_decode(char *s)
+{
+	const char *b64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+	int bit_offset=0, byte_offset, idx, i, n;
+	uint8_t *d = (uint8_t *)s;
+	char *p=NULL;
+
+	n=i=0;
+
+	while (*s && (p=strchr(b64,*s))) {
+		idx = (int)(p - b64);
+		byte_offset = (i*6)/8;
+		bit_offset = (i*6)%8;
+		d[byte_offset] &= ~((1<<(8-bit_offset))-1);
+		if (bit_offset < 3) {
+			d[byte_offset] |= (idx << (2-bit_offset));
+			n = byte_offset+1;
+		} else {
+			d[byte_offset] |= (idx >> (bit_offset-2));
+			d[byte_offset+1] = 0;
+			d[byte_offset+1] |= (idx << (8-(bit_offset-2))) & 0xFF;
+			n = byte_offset+2;
+		}
+		s++; i++;
+	}
+	if (bit_offset >= 3) {
+		n--;
+	}
+
+	if (*s && !p) {
+		/* the only termination allowed */
+		if (*s != '=') {
+			return -1;
+		}
+	}
+
+	/* null terminate */
+	d[n] = 0;
+	return n;
+}
+
+
+/*
+  encode as base64
+  caller frees
+*/
+char *ldb_base64_encode(TALLOC_CTX *mem_ctx, const char *buf, int len)
+{
+	const char *b64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+	int bit_offset, byte_offset, idx, i;
+	const uint8_t *d = (const uint8_t *)buf;
+	int bytes = (len*8 + 5)/6, pad_bytes = (bytes % 4) ? 4 - (bytes % 4) : 0;
+	char *out;
+
+	out = talloc_array(mem_ctx, char, bytes+pad_bytes+1);
+	if (!out) return NULL;
+
+	for (i=0;i<bytes;i++) {
+		byte_offset = (i*6)/8;
+		bit_offset = (i*6)%8;
+		if (bit_offset < 3) {
+			idx = (d[byte_offset] >> (2-bit_offset)) & 0x3F;
+		} else {
+			idx = (d[byte_offset] << (bit_offset-2)) & 0x3F;
+			if (byte_offset+1 < len) {
+				idx |= (d[byte_offset+1] >> (8-(bit_offset-2)));
+			}
+		}
+		out[i] = b64[idx];
+	}
+
+	for (;i<bytes+pad_bytes;i++)
+		out[i] = '=';
+	out[i] = 0;
+
+	return out;
+}
+
+/*
+  see if a buffer should be base64 encoded
+*/
+int ldb_should_b64_encode(struct ldb_context *ldb, const struct ldb_val *val)
+{
+	unsigned int i;
+	uint8_t *p = val->data;
+
+	if (val->length == 0) {
+		return 0;
+	}
+
+	if (p[0] == ' ' || p[0] == ':') {
+		return 1;
+	}
+
+	for (i=0; i<val->length; i++) {
+		if (!isprint(p[i]) || p[i] == '\n') {
+			return 1;
+		}
+	}
+	return 0;
+}
+
+/* this macro is used to handle the return checking on fprintf_fn() */
+#define CHECK_RET do { if (ret < 0) return ret; total += ret; } while (0)
+
+/*
+  write a line folded string onto a file
+*/
+static int fold_string(int (*fprintf_fn)(void *, const char *, ...), void *private_data,
+			const char *buf, size_t length, int start_pos)
+{
+	size_t i;
+	size_t total = 0;
+	int ret;
+
+	for (i=0;i<length;i++) {
+		ret = fprintf_fn(private_data, "%c", buf[i]);
+		CHECK_RET;
+		if (i != (length-1) && (i + start_pos) % 77 == 0) {
+			ret = fprintf_fn(private_data, "\n ");
+			CHECK_RET;
+		}
+	}
+
+	return total;
+}
+
+#undef CHECK_RET
+
+/*
+  encode as base64 to a file
+*/
+static int base64_encode_f(struct ldb_context *ldb,
+			   int (*fprintf_fn)(void *, const char *, ...),
+			   void *private_data,
+			   const char *buf, int len, int start_pos)
+{
+	char *b = ldb_base64_encode(ldb, buf, len);
+	int ret;
+
+	if (!b) {
+		return -1;
+	}
+
+	ret = fold_string(fprintf_fn, private_data, b, strlen(b), start_pos);
+
+	talloc_free(b);
+	return ret;
+}
+
+
+static const struct {
+	const char *name;
+	enum ldb_changetype changetype;
+} ldb_changetypes[] = {
+	{"add",    LDB_CHANGETYPE_ADD},
+	{"delete", LDB_CHANGETYPE_DELETE},
+	{"modify", LDB_CHANGETYPE_MODIFY},
+	{"modrdn", LDB_CHANGETYPE_MODRDN},
+	{"moddn",  LDB_CHANGETYPE_MODRDN},
+	{NULL, 0}
+};
+
+/* this macro is used to handle the return checking on fprintf_fn() */
+#define CHECK_RET do { if (ret < 0) { talloc_free(mem_ctx); return ret; } total += ret; } while (0)
+
+/*
+  write to ldif, using a caller supplied write method, and only printing secrets if we are not in a trace
+*/
+static int ldb_ldif_write_trace(struct ldb_context *ldb,
+				int (*fprintf_fn)(void *, const char *, ...),
+				void *private_data,
+				const struct ldb_ldif *ldif,
+				bool in_trace)
+{
+	TALLOC_CTX *mem_ctx;
+	unsigned int i, j;
+	size_t total = 0;
+	int ret;
+	char *p;
+	const struct ldb_message *msg;
+	const char * const * secret_attributes = ldb_get_opaque(ldb, LDB_SECRET_ATTRIBUTE_LIST_OPAQUE);
+
+	mem_ctx = talloc_named_const(NULL, 0, "ldb_ldif_write");
+
+	msg = ldif->msg;
+	p = ldb_dn_get_extended_linearized(mem_ctx, msg->dn, 1);
+	ret = fprintf_fn(private_data, "dn: %s\n", p);
+	talloc_free(p);
+	CHECK_RET;
+
+	if (ldif->changetype != LDB_CHANGETYPE_NONE) {
+		for (i=0;ldb_changetypes[i].name;i++) {
+			if (ldb_changetypes[i].changetype == ldif->changetype) {
+				break;
+			}
+		}
+		if (!ldb_changetypes[i].name) {
+			ldb_debug(ldb, LDB_DEBUG_ERROR, "Error: Invalid ldif changetype %d",
+				  ldif->changetype);
+			talloc_free(mem_ctx);
+			return -1;
+		}
+		ret = fprintf_fn(private_data, "changetype: %s\n", ldb_changetypes[i].name);
+		CHECK_RET;
+	}
+
+	for (i=0;i<msg->num_elements;i++) {
+		const struct ldb_schema_attribute *a;
+		size_t namelen;
+
+		if (msg->elements[i].name == NULL) {
+			ldb_debug(ldb, LDB_DEBUG_ERROR,
+					"Error: Invalid element name (NULL) at position %d", i);
+			talloc_free(mem_ctx);
+			return -1;
+		}
+
+		namelen = strlen(msg->elements[i].name);
+		a = ldb_schema_attribute_by_name(ldb, msg->elements[i].name);
+
+		if (ldif->changetype == LDB_CHANGETYPE_MODIFY) {
+			switch (msg->elements[i].flags & LDB_FLAG_MOD_MASK) {
+			case LDB_FLAG_MOD_ADD:
+				fprintf_fn(private_data, "add: %s\n",
+					   msg->elements[i].name);
+				break;
+			case LDB_FLAG_MOD_DELETE:
+				fprintf_fn(private_data, "delete: %s\n",
+					   msg->elements[i].name);
+				break;
+			case LDB_FLAG_MOD_REPLACE:
+				fprintf_fn(private_data, "replace: %s\n",
+					   msg->elements[i].name);
+				break;
+			}
+		}
+
+		if (in_trace && secret_attributes && ldb_attr_in_list(secret_attributes, msg->elements[i].name)) {
+			/* Deliberatly skip printing this password */
+			ret = fprintf_fn(private_data, "# %s::: REDACTED SECRET ATTRIBUTE\n",
+					 msg->elements[i].name);
+			CHECK_RET;
+			continue;
+		}
+		for (j=0;j<msg->elements[i].num_values;j++) {
+			struct ldb_val v;
+			bool use_b64_encode = false;
+			bool copy_raw_bytes = false;
+
+			ret = a->syntax->ldif_write_fn(ldb, mem_ctx, &msg->elements[i].values[j], &v);
+			if (ret != LDB_SUCCESS) {
+				v = msg->elements[i].values[j];
+			}
+
+			if (ldb->flags & LDB_FLG_SHOW_BINARY) {
+				use_b64_encode = false;
+				copy_raw_bytes = true;
+			} else if (a->flags & LDB_ATTR_FLAG_FORCE_BASE64_LDIF) {
+				use_b64_encode = true;
+			} else if (msg->elements[i].flags &
+			           LDB_FLAG_FORCE_NO_BASE64_LDIF) {
+				use_b64_encode = false;
+				copy_raw_bytes = true;
+			} else {
+				use_b64_encode = ldb_should_b64_encode(ldb, &v);
+			}
+
+			if (ret != LDB_SUCCESS || use_b64_encode) {
+				ret = fprintf_fn(private_data, "%s:: ",
+						 msg->elements[i].name);
+				CHECK_RET;
+				ret = base64_encode_f(ldb, fprintf_fn, private_data,
+						      (char *)v.data, v.length,
+						      namelen + 3);
+				CHECK_RET;
+				ret = fprintf_fn(private_data, "\n");
+				CHECK_RET;
+			} else {
+				ret = fprintf_fn(private_data, "%s: ", msg->elements[i].name);
+				CHECK_RET;
+				if (copy_raw_bytes) {
+					ret = fprintf_fn(private_data, "%*.*s",
+							 v.length, v.length, (char *)v.data);
+				} else {
+					ret = fold_string(fprintf_fn, private_data,
+							  (char *)v.data, v.length,
+							  namelen + 2);
+				}
+				CHECK_RET;
+				ret = fprintf_fn(private_data, "\n");
+				CHECK_RET;
+			}
+			if (v.data != msg->elements[i].values[j].data) {
+				talloc_free(v.data);
+			}
+		}
+		if (ldif->changetype == LDB_CHANGETYPE_MODIFY) {
+			fprintf_fn(private_data, "-\n");
+		}
+	}
+	ret = fprintf_fn(private_data,"\n");
+	CHECK_RET;
+
+	talloc_free(mem_ctx);
+
+	return total;
+}
+
+#undef CHECK_RET
+
+
+/*
+  write to ldif, using a caller supplied write method
+*/
+int ldb_ldif_write(struct ldb_context *ldb,
+		   int (*fprintf_fn)(void *, const char *, ...),
+		   void *private_data,
+		   const struct ldb_ldif *ldif)
+{
+	return ldb_ldif_write_trace(ldb, fprintf_fn, private_data, ldif, false);
+}
+
+
+/*
+  pull a ldif chunk, which is defined as a piece of data ending in \n\n or EOF
+  this routine removes any RFC2849 continuations and comments
+
+  caller frees
+*/
+static char *next_chunk(struct ldb_context *ldb, TALLOC_CTX *mem_ctx,
+			int (*fgetc_fn)(void *), void *private_data)
+{
+	size_t alloc_size=0, chunk_size = 0;
+	char *chunk = NULL;
+	int c;
+	int in_comment = 0;
+
+	while ((c = fgetc_fn(private_data)) != EOF) {
+		if (chunk_size+1 >= alloc_size) {
+			char *c2;
+			alloc_size += 1024;
+			c2 = talloc_realloc(mem_ctx, chunk, char, alloc_size);
+			if (!c2) {
+				talloc_free(chunk);
+				errno = ENOMEM;
+				return NULL;
+			}
+			chunk = c2;
+		}
+
+		if (in_comment) {
+			if (c == '\n') {
+				in_comment = 0;
+			}
+			continue;
+		}
+
+		/* handle continuation lines - see RFC2849 */
+		if (c == ' ' && chunk_size > 1 && chunk[chunk_size-1] == '\n') {
+			chunk_size--;
+			continue;
+		}
+
+		/* chunks are terminated by a double line-feed */
+		if (c == '\n' && chunk_size > 0 && chunk[chunk_size-1] == '\n') {
+			chunk[chunk_size-1] = 0;
+			return chunk;
+		}
+
+		if (c == '#' && (chunk_size == 0 || chunk[chunk_size-1] == '\n')) {
+			in_comment = 1;
+			continue;
+		}
+
+		/* ignore leading blank lines */
+		if (chunk_size == 0 && c == '\n') {
+			continue;
+		}
+
+		chunk[chunk_size++] = c;
+	}
+
+	if (chunk) {
+		chunk[chunk_size] = 0;
+	}
+
+	return chunk;
+}
+
+
+/* simple ldif attribute parser */
+static int next_attr(TALLOC_CTX *mem_ctx, char **s, const char **attr, struct ldb_val *value)
+{
+	char *p;
+	int base64_encoded = 0;
+	int binary_file = 0;
+
+	if (strncmp(*s, "-\n", 2) == 0) {
+		value->length = 0;
+		*attr = "-";
+		*s += 2;
+		return 0;
+	}
+
+	p = strchr(*s, ':');
+	if (!p) {
+		return -1;
+	}
+
+	*p++ = 0;
+
+	if (*p == ':') {
+		base64_encoded = 1;
+		p++;
+	}
+
+	if (*p == '<') {
+		binary_file = 1;
+		p++;
+	}
+
+	*attr = *s;
+
+	while (*p == ' ' || *p == '\t') {
+		p++;
+	}
+
+	value->data = (uint8_t *)p;
+
+	p = strchr(p, '\n');
+
+	if (!p) {
+		value->length = strlen((char *)value->data);
+		*s = ((char *)value->data) + value->length;
+	} else {
+		value->length = p - (char *)value->data;
+		*s = p+1;
+		*p = 0;
+	}
+
+	if (base64_encoded) {
+		int len = ldb_base64_decode((char *)value->data);
+		if (len == -1) {
+			/* it wasn't valid base64 data */
+			return -1;
+		}
+		value->length = len;
+	}
+
+	if (binary_file) {
+		int len = ldb_read_data_file(mem_ctx, value);
+		if (len == -1) {
+			/* an error occurred while trying to retrieve the file */
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+
+/*
+  free a message from a ldif_read
+*/
+void ldb_ldif_read_free(struct ldb_context *ldb, struct ldb_ldif *ldif)
+{
+	talloc_free(ldif);
+}
+
+int ldb_ldif_parse_modrdn(struct ldb_context *ldb,
+			  const struct ldb_ldif *ldif,
+			  TALLOC_CTX *mem_ctx,
+			  struct ldb_dn **_olddn,
+			  struct ldb_dn **_newrdn,
+			  bool *_deleteoldrdn,
+			  struct ldb_dn **_newsuperior,
+			  struct ldb_dn **_newdn)
+{
+	struct ldb_message *msg = ldif->msg;
+	struct ldb_val *newrdn_val = NULL;
+	struct ldb_val *deleteoldrdn_val = NULL;
+	struct ldb_val *newsuperior_val = NULL;
+	struct ldb_dn *olddn = NULL;
+	struct ldb_dn *newrdn = NULL;
+	bool deleteoldrdn = true;
+	struct ldb_dn *newsuperior = NULL;
+	struct ldb_dn *newdn = NULL;
+	struct ldb_val tmp_false;
+	struct ldb_val tmp_true;
+	bool ok;
+	TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
+
+	if (tmp_ctx == NULL) {
+		ldb_debug(ldb, LDB_DEBUG_FATAL,
+			  "Error: talloc_new() failed");
+		goto err_op;
+	}
+
+	if (ldif->changetype != LDB_CHANGETYPE_MODRDN) {
+		ldb_debug(ldb, LDB_DEBUG_ERROR,
+			  "Error: invalid changetype '%d'",
+			  ldif->changetype);
+		goto err_other;
+	}
+
+	if (msg->num_elements < 2) {
+		ldb_debug(ldb, LDB_DEBUG_ERROR,
+			  "Error: num_elements[%u] < 2",
+			  msg->num_elements);
+		goto err_other;
+	}
+
+	if (msg->num_elements > 3) {
+		ldb_debug(ldb, LDB_DEBUG_ERROR,
+			  "Error: num_elements[%u] > 3",
+			  msg->num_elements);
+		goto err_other;
+	}
+
+#define CHECK_ELEMENT(i, _name, v, needed) do { \
+	v = NULL; \
+	if (msg->num_elements < (i + 1)) { \
+		if (needed) { \
+			ldb_debug(ldb, LDB_DEBUG_ERROR, \
+				  "Error: num_elements[%u] < (%u + 1)", \
+				  msg->num_elements, i); \
+			goto err_other; \
+		} \
+	} else if (ldb_attr_cmp(msg->elements[i].name, _name) != 0) { \
+		ldb_debug(ldb, LDB_DEBUG_ERROR, \
+			  "Error: elements[%u].name[%s] != [%s]", \
+			  i, msg->elements[i].name, _name); \
+		goto err_other; \
+	} else if (msg->elements[i].flags != 0) { \
+		ldb_debug(ldb, LDB_DEBUG_ERROR, \
+			  "Error: elements[%u].flags[0x%X} != [0x0]", \
+			  i, msg->elements[i].flags); \
+		goto err_other; \
+	} else if (msg->elements[i].num_values != 1) { \
+		ldb_debug(ldb, LDB_DEBUG_ERROR, \
+			  "Error: elements[%u].num_values[%u] != 1", \
+			  i, msg->elements[i].num_values); \
+		goto err_other; \
+	} else { \
+		v = &msg->elements[i].values[0]; \
+	} \
+} while (0)
+
+	CHECK_ELEMENT(0, "newrdn", newrdn_val, true);
+	CHECK_ELEMENT(1, "deleteoldrdn", deleteoldrdn_val, true);
+	CHECK_ELEMENT(2, "newsuperior", newsuperior_val, false);
+
+#undef CHECK_ELEMENT
+
+	olddn = ldb_dn_copy(tmp_ctx, msg->dn);
+	if (olddn == NULL) {
+		ldb_debug(ldb, LDB_DEBUG_ERROR,
+			  "Error: failed to copy olddn '%s'",
+			  ldb_dn_get_linearized(msg->dn));
+		goto err_op;
+	}
+
+	newrdn = ldb_dn_from_ldb_val(tmp_ctx, ldb, newrdn_val);
+	if (!ldb_dn_validate(newrdn)) {
+		ldb_debug(ldb, LDB_DEBUG_ERROR,
+			  "Error: Unable to parse dn '%s'",
+			  (char *)newrdn_val->data);
+		goto err_dn;
+	}
+
+	tmp_false.length = 1;
+	tmp_false.data = discard_const_p(uint8_t, "0");
+	tmp_true.length = 1;
+	tmp_true.data = discard_const_p(uint8_t, "1");
+	if (ldb_val_equal_exact(deleteoldrdn_val, &tmp_false) == 1) {
+		deleteoldrdn = false;
+	} else if (ldb_val_equal_exact(deleteoldrdn_val, &tmp_true) == 1) {
+		deleteoldrdn = true;
+	} else {
+		ldb_debug(ldb, LDB_DEBUG_ERROR,
+			  "Error: deleteoldrdn value invalid '%s' not '0'/'1'",
+			  (char *)deleteoldrdn_val->data);
+		goto err_attr;
+	}
+
+	if (newsuperior_val) {
+		newsuperior = ldb_dn_from_ldb_val(tmp_ctx, ldb, newsuperior_val);
+		if (!ldb_dn_validate(newsuperior)) {
+			ldb_debug(ldb, LDB_DEBUG_ERROR,
+				  "Error: Unable to parse dn '%s'",
+				  (char *)newsuperior_val->data);
+			goto err_dn;
+		}
+	} else {
+		newsuperior = ldb_dn_get_parent(tmp_ctx, msg->dn);
+		if (newsuperior == NULL) {
+			ldb_debug(ldb, LDB_DEBUG_ERROR,
+				  "Error: Unable to get parent dn '%s'",
+				  ldb_dn_get_linearized(msg->dn));
+			goto err_dn;
+		}
+	}
+
+	newdn = ldb_dn_copy(tmp_ctx, newrdn);
+	if (newdn == NULL) {
+		ldb_debug(ldb, LDB_DEBUG_ERROR,
+			  "Error: failed to copy newrdn '%s'",
+			  ldb_dn_get_linearized(newrdn));
+		goto err_op;
+	}
+
+	ok = ldb_dn_add_base(newdn, newsuperior);
+	if (!ok) {
+		ldb_debug(ldb, LDB_DEBUG_ERROR,
+			  "Error: failed to base '%s' to newdn '%s'",
+			  ldb_dn_get_linearized(newsuperior),
+			  ldb_dn_get_linearized(newdn));
+		goto err_op;
+	}
+
+	if (_olddn) {
+		*_olddn = talloc_move(mem_ctx, &olddn);
+	}
+	if (_newrdn) {
+		*_newrdn = talloc_move(mem_ctx, &newrdn);
+	}
+	if (_deleteoldrdn) {
+		*_deleteoldrdn = deleteoldrdn;
+	}
+	if (_newsuperior != NULL && _newrdn != NULL) {
+		if (newsuperior_val) {
+			*_newrdn = talloc_move(mem_ctx, &newrdn);
+		} else {
+			*_newrdn = NULL;
+		}
+	}
+	if (_newdn) {
+		*_newdn = talloc_move(mem_ctx, &newdn);
+	}
+
+	talloc_free(tmp_ctx);
+	return LDB_SUCCESS;
+err_other:
+	talloc_free(tmp_ctx);
+	return LDB_ERR_OTHER;
+err_op:
+	talloc_free(tmp_ctx);
+	return LDB_ERR_OPERATIONS_ERROR;
+err_attr:
+	talloc_free(tmp_ctx);
+	return LDB_ERR_INVALID_ATTRIBUTE_SYNTAX;
+err_dn:
+	talloc_free(tmp_ctx);
+	return LDB_ERR_INVALID_DN_SYNTAX;
+}
+
+/*
+ read from a LDIF source, creating a ldb_message
+*/
+struct ldb_ldif *ldb_ldif_read(struct ldb_context *ldb,
+			       int (*fgetc_fn)(void *), void *private_data)
+{
+	struct ldb_ldif *ldif;
+	struct ldb_message *msg;
+	const char *attr=NULL;
+	char *chunk=NULL, *s;
+	struct ldb_val value;
+	unsigned flags = 0;
+	value.data = NULL;
+
+	ldif = talloc(ldb, struct ldb_ldif);
+	if (!ldif) return NULL;
+
+	ldif->msg = ldb_msg_new(ldif);
+	if (ldif->msg == NULL) {
+		talloc_free(ldif);
+		return NULL;
+	}
+
+	ldif->changetype = LDB_CHANGETYPE_NONE;
+	msg = ldif->msg;
+
+	chunk = next_chunk(ldb, ldif, fgetc_fn, private_data);
+	if (!chunk) {
+		goto failed;
+	}
+
+	s = chunk;
+
+	if (next_attr(ldif, &s, &attr, &value) != 0) {
+		goto failed;
+	}
+
+	/* first line must be a dn */
+	if (ldb_attr_cmp(attr, "dn") != 0) {
+		ldb_debug(ldb, LDB_DEBUG_ERROR, "Error: First line of ldif must be a dn not '%s'",
+			  attr);
+		goto failed;
+	}
+
+	msg->dn = ldb_dn_from_ldb_val(msg, ldb, &value);
+
+	if ( ! ldb_dn_validate(msg->dn)) {
+		ldb_debug(ldb, LDB_DEBUG_ERROR, "Error: Unable to parse dn '%s'",
+			  (char *)value.data);
+		goto failed;
+	}
+
+	while (next_attr(ldif, &s, &attr, &value) == 0) {
+		const struct ldb_schema_attribute *a;
+		struct ldb_message_element *el;
+		int ret, empty = 0;
+
+		if (ldb_attr_cmp(attr, "changetype") == 0) {
+			int i;
+			for (i=0;ldb_changetypes[i].name;i++) {
+				if (ldb_attr_cmp((char *)value.data, ldb_changetypes[i].name) == 0) {
+					ldif->changetype = ldb_changetypes[i].changetype;
+					break;
+				}
+			}
+			if (!ldb_changetypes[i].name) {
+				ldb_debug(ldb, LDB_DEBUG_ERROR,
+					  "Error: Bad ldif changetype '%s'",(char *)value.data);
+			}
+			flags = 0;
+			continue;
+		}
+
+		if (ldb_attr_cmp(attr, "add") == 0) {
+			flags = LDB_FLAG_MOD_ADD;
+			empty = 1;
+		}
+		if (ldb_attr_cmp(attr, "delete") == 0) {
+			flags = LDB_FLAG_MOD_DELETE;
+			empty = 1;
+		}
+		if (ldb_attr_cmp(attr, "replace") == 0) {
+			flags = LDB_FLAG_MOD_REPLACE;
+			empty = 1;
+		}
+		if (ldb_attr_cmp(attr, "-") == 0) {
+			flags = 0;
+			continue;
+		}
+
+		if (empty) {
+			if (ldb_msg_add_empty(msg, (char *)value.data, flags, NULL) != 0) {
+				goto failed;
+			}
+			continue;
+		}
+
+		el = &msg->elements[msg->num_elements-1];
+
+		a = ldb_schema_attribute_by_name(ldb, attr);
+
+		if (msg->num_elements > 0 && ldb_attr_cmp(attr, el->name) == 0 &&
+		    flags == el->flags) {
+			/* its a continuation */
+			el->values =
+				talloc_realloc(msg->elements, el->values,
+						 struct ldb_val, el->num_values+1);
+			if (!el->values) {
+				goto failed;
+			}
+			ret = a->syntax->ldif_read_fn(ldb, el->values, &value, &el->values[el->num_values]);
+			if (ret != 0) {
+				goto failed;
+			}
+			if (value.length == 0) {
+				ldb_debug(ldb, LDB_DEBUG_ERROR,
+					  "Error: Attribute value cannot be empty for attribute '%s'", el->name);
+				goto failed;
+			}
+			if (value.data != el->values[el->num_values].data) {
+				talloc_steal(el->values, el->values[el->num_values].data);
+			}
+			el->num_values++;
+		} else {
+			/* its a new attribute */
+			msg->elements = talloc_realloc(msg, msg->elements,
+							 struct ldb_message_element,
+							 msg->num_elements+1);
+			if (!msg->elements) {
+				goto failed;
+			}
+			el = &msg->elements[msg->num_elements];
+			el->flags = flags;
+			el->name = talloc_strdup(msg->elements, attr);
+			el->values = talloc(msg->elements, struct ldb_val);
+			if (!el->values || !el->name) {
+				goto failed;
+			}
+			el->num_values = 1;
+			ret = a->syntax->ldif_read_fn(ldb, el->values, &value, &el->values[0]);
+			if (ret != 0) {
+				goto failed;
+			}
+			if (value.data != el->values[0].data) {
+				talloc_steal(el->values, el->values[0].data);
+			}
+			msg->num_elements++;
+		}
+	}
+
+	if (ldif->changetype == LDB_CHANGETYPE_MODRDN) {
+		int ret;
+
+		ret = ldb_ldif_parse_modrdn(ldb, ldif, ldif,
+					    NULL, NULL, NULL, NULL, NULL);
+		if (ret != LDB_SUCCESS) {
+			goto failed;
+		}
+	}
+
+	return ldif;
+
+failed:
+	talloc_free(ldif);
+	return NULL;
+}
+
+
+
+/*
+  a wrapper around ldif_read() for reading from FILE*
+*/
+
+static int fgetc_file(void *private_data)
+{
+	int c;
+	struct ldif_read_file_state *state =
+		(struct ldif_read_file_state *)private_data;
+	c = fgetc(state->f);
+	if (c == '\n') {
+		state->line_no++;
+	}
+	return c;
+}
+
+struct ldb_ldif *ldb_ldif_read_file_state(struct ldb_context *ldb,
+					  struct ldif_read_file_state *state)
+{
+	return ldb_ldif_read(ldb, fgetc_file, state);
+}
+
+struct ldb_ldif *ldb_ldif_read_file(struct ldb_context *ldb, FILE *f)
+{
+	struct ldif_read_file_state state;
+	state.f = f;
+	return ldb_ldif_read_file_state(ldb, &state);
+}
+
+/*
+  a wrapper around ldif_read() for reading from const char*
+*/
+struct ldif_read_string_state {
+	const char *s;
+};
+
+static int fgetc_string(void *private_data)
+{
+	struct ldif_read_string_state *state =
+		(struct ldif_read_string_state *)private_data;
+	if (state->s[0] != 0) {
+		return *state->s++;
+	}
+	return EOF;
+}
+
+struct ldb_ldif *ldb_ldif_read_string(struct ldb_context *ldb, const char **s)
+{
+	struct ldif_read_string_state state;
+	struct ldb_ldif *ldif;
+	state.s = *s;
+	ldif = ldb_ldif_read(ldb, fgetc_string, &state);
+	*s = state.s;
+	return ldif;
+}
+
+
+/*
+  wrapper around ldif_write() for a file
+*/
+struct ldif_write_file_state {
+	FILE *f;
+};
+
+static int fprintf_file(void *private_data, const char *fmt, ...) PRINTF_ATTRIBUTE(2, 3);
+
+static int fprintf_file(void *private_data, const char *fmt, ...)
+{
+	struct ldif_write_file_state *state =
+		(struct ldif_write_file_state *)private_data;
+	int ret;
+	va_list ap;
+
+	va_start(ap, fmt);
+	ret = vfprintf(state->f, fmt, ap);
+	va_end(ap);
+	return ret;
+}
+
+int ldb_ldif_write_file(struct ldb_context *ldb, FILE *f, const struct ldb_ldif *ldif)
+{
+	struct ldif_write_file_state state;
+	state.f = f;
+	return ldb_ldif_write(ldb, fprintf_file, &state, ldif);
+}
+
+/*
+  wrapper around ldif_write() for a string
+*/
+struct ldif_write_string_state {
+	char *string;
+};
+
+static int ldif_printf_string(void *private_data, const char *fmt, ...) PRINTF_ATTRIBUTE(2, 3);
+
+static int ldif_printf_string(void *private_data, const char *fmt, ...)
+{
+	struct ldif_write_string_state *state =
+		(struct ldif_write_string_state *)private_data;
+	va_list ap;
+	size_t oldlen = talloc_get_size(state->string);
+	va_start(ap, fmt);
+
+	state->string = talloc_vasprintf_append(state->string, fmt, ap);
+	va_end(ap);
+	if (!state->string) {
+		return -1;
+	}
+
+	return talloc_get_size(state->string) - oldlen;
+}
+
+char *ldb_ldif_write_redacted_trace_string(struct ldb_context *ldb, TALLOC_CTX *mem_ctx,
+					   const struct ldb_ldif *ldif)
+{
+	struct ldif_write_string_state state;
+	state.string = talloc_strdup(mem_ctx, "");
+	if (!state.string) {
+		return NULL;
+	}
+	if (ldb_ldif_write_trace(ldb, ldif_printf_string, &state, ldif, true) == -1) {
+		return NULL;
+	}
+	return state.string;
+}
+
+char *ldb_ldif_write_string(struct ldb_context *ldb, TALLOC_CTX *mem_ctx,
+			    const struct ldb_ldif *ldif)
+{
+	struct ldif_write_string_state state;
+	state.string = talloc_strdup(mem_ctx, "");
+	if (!state.string) {
+		return NULL;
+	}
+	if (ldb_ldif_write(ldb, ldif_printf_string, &state, ldif) == -1) {
+		return NULL;
+	}
+	return state.string;
+}
+
+/*
+  convenient function to turn a ldb_message into a string. Useful for
+  debugging
+ */
+char *ldb_ldif_message_string(struct ldb_context *ldb, TALLOC_CTX *mem_ctx,
+			      enum ldb_changetype changetype,
+			      const struct ldb_message *msg)
+{
+	struct ldb_ldif ldif;
+
+	ldif.changetype = changetype;
+	ldif.msg = discard_const_p(struct ldb_message, msg);
+
+	return ldb_ldif_write_string(ldb, mem_ctx, &ldif);
+}
+
+/*
+ * convenient function to turn a ldb_message into a string. Useful for
+ * debugging but also safer if some of the LDIF could be sensitive.
+ *
+ * The secret attributes are specified in a 'const char * const *' within
+ * the LDB_SECRET_ATTRIBUTE_LIST opaque set on the ldb
+ *
+ */
+char *ldb_ldif_message_redacted_string(struct ldb_context *ldb,
+				       TALLOC_CTX *mem_ctx,
+				       enum ldb_changetype changetype,
+				       const struct ldb_message *msg)
+{
+	struct ldb_ldif ldif;
+
+	ldif.changetype = changetype;
+	ldif.msg = discard_const_p(struct ldb_message, msg);
+
+	return ldb_ldif_write_redacted_trace_string(ldb, mem_ctx, &ldif);
+}
diff --git a/lib/ldb/common/ldb_match.c b/lib/ldb/common/ldb_match.c
new file mode 100644
index 0000000..463a24c
--- /dev/null
+++ b/lib/ldb/common/ldb_match.c
@@ -0,0 +1,826 @@
+/* 
+   ldb database library
+
+   Copyright (C) Andrew Tridgell  2004-2005
+   Copyright (C) Simo Sorce            2005
+
+     ** NOTE! The following LGPL license applies to the ldb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+   
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+/*
+ *  Name: ldb
+ *
+ *  Component: ldb expression matching
+ *
+ *  Description: ldb expression matching 
+ *
+ *  Author: Andrew Tridgell
+ */
+
+#include "ldb_private.h"
+#include "dlinklist.h"
+#include "ldb_handlers.h"
+
+/*
+  check if the scope matches in a search result
+*/
+int ldb_match_scope(struct ldb_context *ldb,
+		    struct ldb_dn *base,
+		    struct ldb_dn *dn,
+		    enum ldb_scope scope)
+{
+	int ret = 0;
+
+	if (base == NULL || dn == NULL) {
+		return 1;
+	}
+
+	switch (scope) {
+	case LDB_SCOPE_BASE:
+		if (ldb_dn_compare(base, dn) == 0) {
+			ret = 1;
+		}
+		break;
+
+	case LDB_SCOPE_ONELEVEL:
+		if (ldb_dn_get_comp_num(dn) == (ldb_dn_get_comp_num(base) + 1)) {
+			if (ldb_dn_compare_base(base, dn) == 0) {
+				ret = 1;
+			}
+		}
+		break;
+		
+	case LDB_SCOPE_SUBTREE:
+	default:
+		if (ldb_dn_compare_base(base, dn) == 0) {
+			ret = 1;
+		}
+		break;
+	}
+
+	return ret;
+}
+
+
+/*
+  match if node is present
+*/
+static int ldb_match_present(struct ldb_context *ldb, 
+			     const struct ldb_message *msg,
+			     const struct ldb_parse_tree *tree,
+			     enum ldb_scope scope, bool *matched)
+{
+	const struct ldb_schema_attribute *a;
+	struct ldb_message_element *el;
+
+	if (ldb_attr_dn(tree->u.present.attr) == 0) {
+		*matched = true;
+		return LDB_SUCCESS;
+	}
+
+	el = ldb_msg_find_element(msg, tree->u.present.attr);
+	if (el == NULL) {
+		*matched = false;
+		return LDB_SUCCESS;
+	}
+
+	a = ldb_schema_attribute_by_name(ldb, el->name);
+	if (!a) {
+		return LDB_ERR_INVALID_ATTRIBUTE_SYNTAX;
+	}
+
+	if (a->syntax->operator_fn) {
+		unsigned int i;
+		for (i = 0; i < el->num_values; i++) {
+			int ret = a->syntax->operator_fn(ldb, LDB_OP_PRESENT, a, &el->values[i], NULL, matched);
+			if (ret != LDB_SUCCESS) return ret;
+			if (*matched) return LDB_SUCCESS;
+		}
+		*matched = false;
+		return LDB_SUCCESS;
+	}
+
+	*matched = true;
+	return LDB_SUCCESS;
+}
+
+static int ldb_match_comparison(struct ldb_context *ldb, 
+				const struct ldb_message *msg,
+				const struct ldb_parse_tree *tree,
+				enum ldb_scope scope,
+				enum ldb_parse_op comp_op, bool *matched)
+{
+	unsigned int i;
+	struct ldb_message_element *el;
+	const struct ldb_schema_attribute *a;
+
+	/* FIXME: APPROX comparison not handled yet */
+	if (comp_op == LDB_OP_APPROX) {
+		return LDB_ERR_INAPPROPRIATE_MATCHING;
+	}
+
+	el = ldb_msg_find_element(msg, tree->u.comparison.attr);
+	if (el == NULL) {
+		*matched = false;
+		return LDB_SUCCESS;
+	}
+
+	a = ldb_schema_attribute_by_name(ldb, el->name);
+	if (!a) {
+		return LDB_ERR_INVALID_ATTRIBUTE_SYNTAX;
+	}
+
+	for (i = 0; i < el->num_values; i++) {
+		if (a->syntax->operator_fn) {
+			int ret;
+			ret = a->syntax->operator_fn(ldb, comp_op, a, &el->values[i], &tree->u.comparison.value, matched);
+			if (ret != LDB_SUCCESS) return ret;
+			if (*matched) return LDB_SUCCESS;
+		} else {
+			int ret = a->syntax->comparison_fn(ldb, ldb, &el->values[i], &tree->u.comparison.value);
+
+			if (ret == 0) {
+				*matched = true;
+				return LDB_SUCCESS;
+			}
+			if (ret > 0 && comp_op == LDB_OP_GREATER) {
+				*matched = true;
+				return LDB_SUCCESS;
+			}
+			if (ret < 0 && comp_op == LDB_OP_LESS) {
+				*matched = true;
+				return LDB_SUCCESS;
+			}
+		}
+	}
+
+	*matched = false;
+	return LDB_SUCCESS;
+}
+
+/*
+  match a simple leaf node
+*/
+static int ldb_match_equality(struct ldb_context *ldb, 
+			      const struct ldb_message *msg,
+			      const struct ldb_parse_tree *tree,
+			      enum ldb_scope scope,
+			      bool *matched)
+{
+	unsigned int i;
+	struct ldb_message_element *el;
+	const struct ldb_schema_attribute *a;
+	struct ldb_dn *valuedn;
+	int ret;
+
+	if (ldb_attr_dn(tree->u.equality.attr) == 0) {
+		valuedn = ldb_dn_from_ldb_val(ldb, ldb, &tree->u.equality.value);
+		if (valuedn == NULL) {
+			return LDB_ERR_INVALID_DN_SYNTAX;
+		}
+
+		ret = ldb_dn_compare(msg->dn, valuedn);
+
+		talloc_free(valuedn);
+
+		*matched = (ret == 0);
+		return LDB_SUCCESS;
+	}
+
+	/* TODO: handle the "*" case derived from an extended search
+	   operation without the attibute type defined */
+	el = ldb_msg_find_element(msg, tree->u.equality.attr);
+	if (el == NULL) {
+		*matched = false;
+		return LDB_SUCCESS;
+	}
+
+	a = ldb_schema_attribute_by_name(ldb, el->name);
+	if (a == NULL) {
+		return LDB_ERR_INVALID_ATTRIBUTE_SYNTAX;
+	}
+
+	for (i=0;i<el->num_values;i++) {
+		if (a->syntax->operator_fn) {
+			ret = a->syntax->operator_fn(ldb, LDB_OP_EQUALITY, a,
+						     &tree->u.equality.value, &el->values[i], matched);
+			if (ret != LDB_SUCCESS) return ret;
+			if (*matched) return LDB_SUCCESS;
+		} else {
+			if (a->syntax->comparison_fn(ldb, ldb, &tree->u.equality.value,
+						     &el->values[i]) == 0) {
+				*matched = true;
+				return LDB_SUCCESS;
+			}
+		}
+	}
+
+	*matched = false;
+	return LDB_SUCCESS;
+}
+
+static int ldb_wildcard_compare(struct ldb_context *ldb,
+				const struct ldb_parse_tree *tree,
+				const struct ldb_val value, bool *matched)
+{
+	const struct ldb_schema_attribute *a;
+	struct ldb_val val;
+	struct ldb_val cnk;
+	struct ldb_val *chunk;
+	uint8_t *save_p = NULL;
+	unsigned int c = 0;
+
+	if (tree->operation != LDB_OP_SUBSTRING) {
+		*matched = false;
+		return LDB_ERR_INAPPROPRIATE_MATCHING;
+	}
+
+	a = ldb_schema_attribute_by_name(ldb, tree->u.substring.attr);
+	if (!a) {
+		return LDB_ERR_INVALID_ATTRIBUTE_SYNTAX;
+	}
+
+	if (tree->u.substring.chunks == NULL) {
+		*matched = false;
+		return LDB_SUCCESS;
+	}
+
+	/* No need to just copy this value for a binary match */
+	if (a->syntax->canonicalise_fn != ldb_handler_copy) {
+		if (a->syntax->canonicalise_fn(ldb, ldb, &value, &val) != 0) {
+			return LDB_ERR_INVALID_ATTRIBUTE_SYNTAX;
+		}
+
+		/*
+		 * Only set save_p if we allocate (call
+		 * a->syntax->canonicalise_fn()), as we
+		 * talloc_free(save_p) below to clean up
+		 */
+		save_p = val.data;
+	} else {
+		val = value;
+	}
+
+	cnk.data = NULL;
+
+	if ( ! tree->u.substring.start_with_wildcard ) {
+		uint8_t *cnk_to_free = NULL;
+
+		chunk = tree->u.substring.chunks[c];
+		/* No need to just copy this value for a binary match */
+		if (a->syntax->canonicalise_fn != ldb_handler_copy) {
+			if (a->syntax->canonicalise_fn(ldb, ldb, chunk, &cnk) != 0) {
+				goto mismatch;
+			}
+
+			cnk_to_free = cnk.data;
+		} else {
+			cnk = *chunk;
+		}
+
+		/* This deals with wildcard prefix searches on binary attributes (eg objectGUID) */
+		if (cnk.length > val.length) {
+			TALLOC_FREE(cnk_to_free);
+			goto mismatch;
+		}
+		/*
+		 * Empty strings are returned as length 0. Ensure
+		 * we can cope with this.
+		 */
+		if (cnk.length == 0) {
+			TALLOC_FREE(cnk_to_free);
+			goto mismatch;
+		}
+
+		if (memcmp((char *)val.data, (char *)cnk.data, cnk.length) != 0) {
+			TALLOC_FREE(cnk_to_free);
+			goto mismatch;
+		}
+
+		val.length -= cnk.length;
+		val.data += cnk.length;
+		c++;
+		TALLOC_FREE(cnk_to_free);
+		cnk.data = NULL;
+	}
+
+	while (tree->u.substring.chunks[c]) {
+		uint8_t *p;
+		uint8_t *cnk_to_free = NULL;
+
+		chunk = tree->u.substring.chunks[c];
+		/* No need to just copy this value for a binary match */
+		if (a->syntax->canonicalise_fn != ldb_handler_copy) {
+			if (a->syntax->canonicalise_fn(ldb, ldb, chunk, &cnk) != 0) {
+				goto mismatch;
+			}
+
+			cnk_to_free = cnk.data;
+		} else {
+			cnk = *chunk;
+		}
+		/*
+		 * Empty strings are returned as length 0. Ensure
+		 * we can cope with this.
+		 */
+		if (cnk.length == 0) {
+			TALLOC_FREE(cnk_to_free);
+			goto mismatch;
+		}
+		if (cnk.length > val.length) {
+			TALLOC_FREE(cnk_to_free);
+			goto mismatch;
+		}
+
+		if ( (tree->u.substring.chunks[c + 1]) == NULL &&
+		     (! tree->u.substring.end_with_wildcard) ) {
+			/*
+			 * The last bit, after all the asterisks, must match
+			 * exactly the last bit of the string.
+			 */
+			int cmp;
+			p = val.data + val.length - cnk.length;
+			cmp = memcmp(p,
+				     cnk.data,
+				     cnk.length);
+			TALLOC_FREE(cnk_to_free);
+
+			if (cmp != 0) {
+				goto mismatch;
+			}
+		} else {
+			/*
+			 * Values might be binary blobs. Don't use string
+			 * search, but memory search instead.
+			 */
+			p = memmem((const void *)val.data, val.length,
+				   (const void *)cnk.data, cnk.length);
+			if (p == NULL) {
+				TALLOC_FREE(cnk_to_free);
+				goto mismatch;
+			}
+			/* move val to the end of the match */
+			p += cnk.length;
+			val.length -= (p - val.data);
+			val.data = p;
+			TALLOC_FREE(cnk_to_free);
+		}
+		c++;
+	}
+
+	talloc_free(save_p);
+	*matched = true;
+	return LDB_SUCCESS;
+
+mismatch:
+	*matched = false;
+	talloc_free(save_p);
+	return LDB_SUCCESS;
+}
+
+/*
+  match a simple leaf node
+*/
+static int ldb_match_substring(struct ldb_context *ldb, 
+			       const struct ldb_message *msg,
+			       const struct ldb_parse_tree *tree,
+			       enum ldb_scope scope, bool *matched)
+{
+	unsigned int i;
+	struct ldb_message_element *el;
+
+	el = ldb_msg_find_element(msg, tree->u.substring.attr);
+	if (el == NULL) {
+		*matched = false;
+		return LDB_SUCCESS;
+	}
+
+	for (i = 0; i < el->num_values; i++) {
+		int ret;
+		ret = ldb_wildcard_compare(ldb, tree, el->values[i], matched);
+		if (ret != LDB_SUCCESS) return ret;
+		if (*matched) return LDB_SUCCESS;
+	}
+
+	*matched = false;
+	return LDB_SUCCESS;
+}
+
+
+/*
+  bitwise and/or comparator depending on oid
+*/
+static int ldb_comparator_bitmask(const char *oid, const struct ldb_val *v1, const struct ldb_val *v2,
+				  bool *matched)
+{
+	uint64_t i1, i2;
+	char ibuf[100];
+	char *endptr = NULL;
+
+	if (v1->length >= sizeof(ibuf)-1) {
+		return LDB_ERR_INVALID_ATTRIBUTE_SYNTAX;
+	}
+	memcpy(ibuf, (char *)v1->data, v1->length);
+	ibuf[v1->length] = 0;
+	i1 = strtoull(ibuf, &endptr, 0);
+	if (endptr != NULL) {
+		if (endptr == ibuf || *endptr != 0) {
+			return LDB_ERR_INVALID_ATTRIBUTE_SYNTAX;
+		}
+	}
+
+	if (v2->length >= sizeof(ibuf)-1) {
+		return LDB_ERR_INVALID_ATTRIBUTE_SYNTAX;
+	}
+	endptr = NULL;
+	memcpy(ibuf, (char *)v2->data, v2->length);
+	ibuf[v2->length] = 0;
+	i2 = strtoull(ibuf, &endptr, 0);
+	if (endptr != NULL) {
+		if (endptr == ibuf || *endptr != 0) {
+			return LDB_ERR_INVALID_ATTRIBUTE_SYNTAX;
+		}
+	}
+	if (strcmp(LDB_OID_COMPARATOR_AND, oid) == 0) {
+		*matched = ((i1 & i2) == i2);
+	} else if (strcmp(LDB_OID_COMPARATOR_OR, oid) == 0) {
+		*matched = ((i1 & i2) != 0);
+	} else {
+		return LDB_ERR_INAPPROPRIATE_MATCHING;
+	}
+	return LDB_SUCCESS;
+}
+
+static int ldb_match_bitmask(struct ldb_context *ldb,
+			     const char *oid,
+			     const struct ldb_message *msg,
+			     const char *attribute_to_match,
+			     const struct ldb_val *value_to_match,
+			     bool *matched)
+{
+	unsigned int i;
+	struct ldb_message_element *el;
+
+	/* find the message element */
+	el = ldb_msg_find_element(msg, attribute_to_match);
+	if (el == NULL) {
+		*matched = false;
+		return LDB_SUCCESS;
+	}
+
+	for (i=0;i<el->num_values;i++) {
+		int ret;
+		struct ldb_val *v = &el->values[i];
+
+		ret = ldb_comparator_bitmask(oid, v, value_to_match, matched);
+		if (ret != LDB_SUCCESS) {
+			return ret;
+		}
+		if (*matched) {
+			return LDB_SUCCESS;
+		}
+	}
+
+	*matched = false;
+	return LDB_SUCCESS;
+}
+
+/*
+  always return false
+*/
+static int ldb_comparator_false(struct ldb_context *ldb,
+				const char *oid,
+				const struct ldb_message *msg,
+				const char *attribute_to_match,
+				const struct ldb_val *value_to_match,
+				bool *matched)
+{
+	*matched = false;
+	return LDB_SUCCESS;
+}
+
+
+static const struct ldb_extended_match_rule *ldb_find_extended_match_rule(struct ldb_context *ldb,
+									  const char *oid)
+{
+	struct ldb_extended_match_entry *extended_match_rule;
+
+	for (extended_match_rule = ldb->extended_match_rules;
+	     extended_match_rule;
+	     extended_match_rule = extended_match_rule->next) {
+		if (strcmp(extended_match_rule->rule->oid, oid) == 0) {
+			return extended_match_rule->rule;
+		}
+	}
+
+	return NULL;
+}
+
+
+/*
+  extended match, handles things like bitops
+*/
+static int ldb_match_extended(struct ldb_context *ldb, 
+			      const struct ldb_message *msg,
+			      const struct ldb_parse_tree *tree,
+			      enum ldb_scope scope, bool *matched)
+{
+	const struct ldb_extended_match_rule *rule;
+
+	if (tree->u.extended.dnAttributes) {
+		/* FIXME: We really need to find out what this ":dn" part in
+		 * an extended match means and how to handle it. For now print
+		 * only a warning to have s3 winbind and other tools working
+		 * against us. - Matthias */
+		ldb_debug(ldb, LDB_DEBUG_WARNING, "ldb: dnAttributes extended match not supported yet");
+	}
+	if (tree->u.extended.rule_id == NULL) {
+		ldb_debug(ldb, LDB_DEBUG_ERROR, "ldb: no-rule extended matches not supported yet");
+		return LDB_ERR_INAPPROPRIATE_MATCHING;
+	}
+	if (tree->u.extended.attr == NULL) {
+		ldb_debug(ldb, LDB_DEBUG_ERROR, "ldb: no-attribute extended matches not supported yet");
+		return LDB_ERR_INAPPROPRIATE_MATCHING;
+	}
+
+	rule = ldb_find_extended_match_rule(ldb, tree->u.extended.rule_id);
+	if (rule == NULL) {
+		*matched = false;
+		ldb_debug(ldb, LDB_DEBUG_ERROR, "ldb: unknown extended rule_id %s",
+			  tree->u.extended.rule_id);
+		return LDB_SUCCESS;
+	}
+
+	return rule->callback(ldb, rule->oid, msg,
+			      tree->u.extended.attr,
+			      &tree->u.extended.value, matched);
+}
+
+static bool ldb_must_suppress_match(const struct ldb_message *msg,
+				    const struct ldb_parse_tree *tree)
+{
+	const char *attr = NULL;
+	struct ldb_message_element *el = NULL;
+
+	attr = ldb_parse_tree_get_attr(tree);
+	if (attr == NULL) {
+		return false;
+	}
+
+	/* find the message element */
+	el = ldb_msg_find_element(msg, attr);
+	if (el == NULL) {
+		return false;
+	}
+
+	return ldb_msg_element_is_inaccessible(el);
+}
+
+/*
+  Check if a particular message will match the given filter
+
+  set *matched to true if it matches, false otherwise
+
+  returns LDB_SUCCESS or an error
+
+  this is a recursive function, and does short-circuit evaluation
+ */
+int ldb_match_message(struct ldb_context *ldb,
+		      const struct ldb_message *msg,
+		      const struct ldb_parse_tree *tree,
+		      enum ldb_scope scope, bool *matched)
+{
+	unsigned int i;
+	int ret;
+
+	*matched = false;
+
+	if (scope != LDB_SCOPE_BASE && ldb_dn_is_special(msg->dn)) {
+		/* don't match special records except on base searches */
+		return LDB_SUCCESS;
+	}
+
+	/*
+	 * Suppress matches on confidential attributes (handled
+	 * manually in extended matches as these can do custom things
+	 * like read other parts of the DB or other attributes).
+	 */
+	if (tree->operation != LDB_OP_EXTENDED) {
+		if (ldb_must_suppress_match(msg, tree)) {
+			return LDB_SUCCESS;
+		}
+	}
+
+	switch (tree->operation) {
+	case LDB_OP_AND:
+		for (i=0;i<tree->u.list.num_elements;i++) {
+			ret = ldb_match_message(ldb, msg, tree->u.list.elements[i], scope, matched);
+			if (ret != LDB_SUCCESS) return ret;
+			if (!*matched) return LDB_SUCCESS;
+		}
+		*matched = true;
+		return LDB_SUCCESS;
+
+	case LDB_OP_OR:
+		for (i=0;i<tree->u.list.num_elements;i++) {
+			ret = ldb_match_message(ldb, msg, tree->u.list.elements[i], scope, matched);
+			if (ret != LDB_SUCCESS) return ret;
+			if (*matched) return LDB_SUCCESS;
+		}
+		*matched = false;
+		return LDB_SUCCESS;
+
+	case LDB_OP_NOT:
+		ret = ldb_match_message(ldb, msg, tree->u.isnot.child, scope, matched);
+		if (ret != LDB_SUCCESS) return ret;
+		*matched = ! *matched;
+		return LDB_SUCCESS;
+
+	case LDB_OP_EQUALITY:
+		return ldb_match_equality(ldb, msg, tree, scope, matched);
+
+	case LDB_OP_SUBSTRING:
+		return ldb_match_substring(ldb, msg, tree, scope, matched);
+
+	case LDB_OP_GREATER:
+		return ldb_match_comparison(ldb, msg, tree, scope, LDB_OP_GREATER, matched);
+
+	case LDB_OP_LESS:
+		return ldb_match_comparison(ldb, msg, tree, scope, LDB_OP_LESS, matched);
+
+	case LDB_OP_PRESENT:
+		return ldb_match_present(ldb, msg, tree, scope, matched);
+
+	case LDB_OP_APPROX:
+		return ldb_match_comparison(ldb, msg, tree, scope, LDB_OP_APPROX, matched);
+
+	case LDB_OP_EXTENDED:
+		return ldb_match_extended(ldb, msg, tree, scope, matched);
+	}
+
+	return LDB_ERR_INAPPROPRIATE_MATCHING;
+}
+
+/*
+  return 0 if the given parse tree matches the given message. Assumes
+  the message is in sorted order
+
+  return 1 if it matches, and 0 if it doesn't match
+*/
+
+int ldb_match_msg(struct ldb_context *ldb,
+		  const struct ldb_message *msg,
+		  const struct ldb_parse_tree *tree,
+		  struct ldb_dn *base,
+		  enum ldb_scope scope)
+{
+	bool matched;
+	int ret;
+
+	if ( ! ldb_match_scope(ldb, base, msg->dn, scope) ) {
+		return 0;
+	}
+
+	ret = ldb_match_message(ldb, msg, tree, scope, &matched);
+	if (ret != LDB_SUCCESS) {
+		/* to match the old API, we need to consider this a
+		   failure to match */
+		return 0;
+	}
+	return matched?1:0;
+}
+
+int ldb_match_msg_error(struct ldb_context *ldb,
+			const struct ldb_message *msg,
+			const struct ldb_parse_tree *tree,
+			struct ldb_dn *base,
+			enum ldb_scope scope,
+			bool *matched)
+{
+	if ( ! ldb_match_scope(ldb, base, msg->dn, scope) ) {
+		*matched = false;
+		return LDB_SUCCESS;
+	}
+
+	return ldb_match_message(ldb, msg, tree, scope, matched);
+}
+
+int ldb_match_msg_objectclass(const struct ldb_message *msg,
+			      const char *objectclass)
+{
+	unsigned int i;
+	struct ldb_message_element *el = ldb_msg_find_element(msg, "objectClass");
+	if (!el) {
+		return 0;
+	}
+	for (i=0; i < el->num_values; i++) {
+		if (ldb_attr_cmp((const char *)el->values[i].data, objectclass) == 0) {
+			return 1;
+		}
+	}
+	return 0;
+}
+
+_PRIVATE_ int ldb_register_extended_match_rules(struct ldb_context *ldb)
+{
+	struct ldb_extended_match_rule *bitmask_and;
+	struct ldb_extended_match_rule *bitmask_or;
+	struct ldb_extended_match_rule *always_false;
+	int ret;
+
+	/* Register bitmask-and match */
+	bitmask_and = talloc_zero(ldb, struct ldb_extended_match_rule);
+	if (bitmask_and == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	bitmask_and->oid = LDB_OID_COMPARATOR_AND;
+	bitmask_and->callback = ldb_match_bitmask;
+
+	ret = ldb_register_extended_match_rule(ldb, bitmask_and);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	/* Register bitmask-or match */
+	bitmask_or = talloc_zero(ldb, struct ldb_extended_match_rule);
+	if (bitmask_or == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	bitmask_or->oid = LDB_OID_COMPARATOR_OR;
+	bitmask_or->callback = ldb_match_bitmask;
+
+	ret = ldb_register_extended_match_rule(ldb, bitmask_or);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	/* Register always-false match */
+	always_false = talloc_zero(ldb, struct ldb_extended_match_rule);
+	if (always_false == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	always_false->oid = SAMBA_LDAP_MATCH_ALWAYS_FALSE;
+	always_false->callback = ldb_comparator_false;
+
+	ret = ldb_register_extended_match_rule(ldb, always_false);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	return LDB_SUCCESS;
+}
+
+/*
+  register a new ldb extended matching rule
+*/
+int ldb_register_extended_match_rule(struct ldb_context *ldb,
+				     const struct ldb_extended_match_rule *rule)
+{
+	const struct ldb_extended_match_rule *lookup_rule;
+	struct ldb_extended_match_entry *entry;
+
+	lookup_rule = ldb_find_extended_match_rule(ldb, rule->oid);
+	if (lookup_rule) {
+		return LDB_ERR_ENTRY_ALREADY_EXISTS;
+	}
+
+	entry = talloc_zero(ldb, struct ldb_extended_match_entry);
+	if (!entry) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	entry->rule = rule;
+	DLIST_ADD_END(ldb->extended_match_rules, entry);
+
+	return LDB_SUCCESS;
+}
+
+int ldb_register_redact_callback(struct ldb_context *ldb,
+				 ldb_redact_fn redact_fn,
+				 struct ldb_module *module)
+{
+	if (ldb->redact.callback != NULL) {
+		return LDB_ERR_ENTRY_ALREADY_EXISTS;
+	}
+
+	ldb->redact.callback = redact_fn;
+	ldb->redact.module = module;
+	return LDB_SUCCESS;
+}
diff --git a/lib/ldb/common/ldb_modules.c b/lib/ldb/common/ldb_modules.c
new file mode 100644
index 0000000..2d0494a
--- /dev/null
+++ b/lib/ldb/common/ldb_modules.c
@@ -0,0 +1,1242 @@
+/*
+   ldb database library
+
+   Copyright (C) Simo Sorce  2004-2008
+
+     ** NOTE! The following LGPL license applies to the ldb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+/*
+ *  Name: ldb
+ *
+ *  Component: ldb modules core
+ *
+ *  Description: core modules routines
+ *
+ *  Author: Simo Sorce
+ */
+
+#include "ldb_private.h"
+#include "dlinklist.h"
+#include "system/dir.h"
+
+static char *ldb_modules_strdup_no_spaces(TALLOC_CTX *mem_ctx, const char *string)
+{
+	size_t i, len;
+	char *trimmed;
+
+	trimmed = talloc_strdup(mem_ctx, string);
+	if (!trimmed) {
+		return NULL;
+	}
+
+	len = strlen(trimmed);
+	for (i = 0; trimmed[i] != '\0'; i++) {
+		switch (trimmed[i]) {
+		case ' ':
+		case '\t':
+		case '\n':
+			memmove(&trimmed[i], &trimmed[i + 1], len -i -1);
+			break;
+		}
+	}
+
+	return trimmed;
+}
+
+
+/* modules are called in inverse order on the stack.
+   Lets place them as an admin would think the right order is.
+   Modules order is important */
+const char **ldb_modules_list_from_string(struct ldb_context *ldb, TALLOC_CTX *mem_ctx, const char *string)
+{
+	char **modules = NULL;
+	const char **m;
+	char *modstr, *p;
+	unsigned int i;
+
+	/* spaces not admitted */
+	modstr = ldb_modules_strdup_no_spaces(mem_ctx, string);
+	if ( ! modstr) {
+		ldb_debug(ldb, LDB_DEBUG_FATAL, "Out of Memory in ldb_modules_strdup_no_spaces()");
+		return NULL;
+	}
+
+	modules = talloc_realloc(mem_ctx, modules, char *, 2);
+	if ( ! modules ) {
+		ldb_debug(ldb, LDB_DEBUG_FATAL, "Out of Memory in ldb_modules_list_from_string()");
+		talloc_free(modstr);
+		return NULL;
+	}
+	talloc_steal(modules, modstr);
+
+	if (modstr[0] == '\0') {
+		modules[0] = NULL;
+		m = discard_const_p(const char *, modules);
+		return m;
+	}
+
+	i = 0;
+	/* The str*r*chr walks backwards:  This is how we get the inverse order mentioned above */
+	while ((p = strrchr(modstr, ',')) != NULL) {
+		*p = '\0';
+		p++;
+		modules[i] = p;
+
+		i++;
+		modules = talloc_realloc(mem_ctx, modules, char *, i + 2);
+		if ( ! modules ) {
+			ldb_debug(ldb, LDB_DEBUG_FATAL, "Out of Memory in ldb_modules_list_from_string()");
+			return NULL;
+		}
+
+	}
+	modules[i] = modstr;
+
+	modules[i + 1] = NULL;
+
+	m = discard_const_p(const char *, modules);
+
+	return m;
+}
+
+static struct backends_list_entry {
+	struct ldb_backend_ops *ops;
+	struct backends_list_entry *prev, *next;
+} *ldb_backends = NULL;
+
+static struct ops_list_entry {
+	const struct ldb_module_ops *ops;
+	struct ops_list_entry *next;
+} *registered_modules = NULL;
+
+static struct backends_list_entry *ldb_find_backend(const char *url_prefix)
+{
+	struct backends_list_entry *backend;
+
+	for (backend = ldb_backends; backend; backend = backend->next) {
+		if (strcmp(backend->ops->name, url_prefix) == 0) {
+			return backend;
+		}
+	}
+
+	return NULL;
+}
+
+/*
+  register a new ldb backend
+
+  if override is true, then override any existing backend for this prefix
+*/
+int ldb_register_backend(const char *url_prefix, ldb_connect_fn connectfn, bool override)
+{
+	struct backends_list_entry *be;
+
+	be = ldb_find_backend(url_prefix);
+	if (be) {
+		if (!override) {
+			return LDB_SUCCESS;
+		}
+	} else {
+		be = talloc_zero(ldb_backends, struct backends_list_entry);
+		if (!be) {
+			return LDB_ERR_OPERATIONS_ERROR;
+		}
+		be->ops = talloc_zero(be, struct ldb_backend_ops);
+		if (!be->ops) {
+			talloc_free(be);
+			return LDB_ERR_OPERATIONS_ERROR;
+		}
+		DLIST_ADD_END(ldb_backends, be);
+	}
+
+	be->ops->name = url_prefix;
+	be->ops->connect_fn = connectfn;
+
+	return LDB_SUCCESS;
+}
+
+/*
+   Return the ldb module form of a database.
+   The URL looks something like this:
+     tdb://PATH
+     ldb://PATH
+     mdb://PATH
+     ldapi://PATH
+     PATH          (unadorned PATH defaults to tdb://)
+
+   for a complete list of backends (including possibly unmaintained ones) grep
+   for calls to ldb_register_backend().
+
+   the options are passed uninterpreted to the backend, and are
+   backend specific.
+
+   This allows modules to get at only the backend module, for example where a
+   module may wish to direct certain requests at a particular backend.
+*/
+int ldb_module_connect_backend(struct ldb_context *ldb,
+			       const char *url,
+			       const char *options[],
+			       struct ldb_module **backend_module)
+{
+	int ret;
+	char *backend;
+	struct backends_list_entry *be;
+	char *colon = NULL;
+
+	colon = strchr(url, ':');
+	if (colon != NULL) {
+		backend = talloc_strndup(ldb, url, colon-url);
+	} else {
+		/* Default to tdb */
+		backend = talloc_strdup(ldb, "tdb");
+	}
+	if (backend == NULL) {
+		return ldb_oom(ldb);
+	}
+
+	be = ldb_find_backend(backend);
+
+	talloc_free(backend);
+
+	if (be == NULL) {
+		ldb_debug(ldb, LDB_DEBUG_FATAL,
+			  "Unable to find backend for '%s' - do you need to set LDB_MODULES_PATH?", url);
+		return LDB_ERR_OTHER;
+	}
+
+	ret = be->ops->connect_fn(ldb, url, ldb->flags, options, backend_module);
+
+	if (ret != LDB_SUCCESS) {
+		ldb_debug(ldb, LDB_DEBUG_ERROR,
+			  "Failed to connect to '%s' with backend '%s': %s", url, be->ops->name, ldb_errstring(ldb));
+		return ret;
+	}
+	return ret;
+}
+
+static struct ldb_hooks {
+	struct ldb_hooks *next, *prev;
+	ldb_hook_fn hook_fn;
+} *ldb_hooks;
+
+/*
+  register a ldb hook function
+ */
+int ldb_register_hook(ldb_hook_fn hook_fn)
+{
+	struct ldb_hooks *lc;
+	lc = talloc_zero(ldb_hooks, struct ldb_hooks);
+	if (lc == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	lc->hook_fn = hook_fn;
+	DLIST_ADD_END(ldb_hooks, lc);
+	return LDB_SUCCESS;
+}
+
+/*
+  call ldb hooks of a given type
+ */
+int ldb_modules_hook(struct ldb_context *ldb, enum ldb_module_hook_type t)
+{
+	struct ldb_hooks *lc;
+	for (lc = ldb_hooks; lc; lc=lc->next) {
+		int ret = lc->hook_fn(ldb, t);
+		if (ret != LDB_SUCCESS) {
+			return ret;
+		}
+	}
+	return LDB_SUCCESS;
+}
+
+
+static const struct ldb_module_ops *ldb_find_module_ops(const char *name)
+{
+	struct ops_list_entry *e;
+
+	for (e = registered_modules; e; e = e->next) {
+ 		if (strcmp(e->ops->name, name) == 0)
+			return e->ops;
+	}
+
+	return NULL;
+}
+
+
+int ldb_register_module(const struct ldb_module_ops *ops)
+{
+	struct ops_list_entry *entry;
+
+	if (ldb_find_module_ops(ops->name) != NULL)
+		return LDB_ERR_ENTRY_ALREADY_EXISTS;
+
+	/*
+	 * ldb modules are not (yet) unloaded and
+	 * are only loaded once (the above check
+	 * makes sure of this). Allocate off the NULL
+	 * context. We never want this to be freed
+	 * until process shutdown. If eventually we
+	 * want to unload ldb modules we can add a
+	 * deregister function that walks and
+	 * frees the list.
+	 */
+	entry = talloc(NULL, struct ops_list_entry);
+	if (entry == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	entry->ops = ops;
+	entry->next = registered_modules;
+	registered_modules = entry;
+
+	return LDB_SUCCESS;
+}
+
+/*
+  load a list of modules
+ */
+int ldb_module_load_list(struct ldb_context *ldb, const char **module_list,
+			 struct ldb_module *backend, struct ldb_module **out)
+{
+	struct ldb_module *module;
+	unsigned int i;
+
+	module = backend;
+
+	for (i = 0; module_list && module_list[i] != NULL; i++) {
+		struct ldb_module *current;
+		const struct ldb_module_ops *ops;
+
+		if (strcmp(module_list[i], "") == 0) {
+			continue;
+		}
+
+		ops = ldb_find_module_ops(module_list[i]);
+
+		if (ops == NULL) {
+			ldb_debug(ldb, LDB_DEBUG_FATAL, "WARNING: Module [%s] not found - do you need to set LDB_MODULES_PATH?",
+				  module_list[i]);
+			return LDB_ERR_OPERATIONS_ERROR;
+		}
+
+		current = talloc_zero(ldb, struct ldb_module);
+		if (current == NULL) {
+			return LDB_ERR_OPERATIONS_ERROR;
+		}
+		talloc_set_name(current, "ldb_module: %s", module_list[i]);
+
+		current->ldb = ldb;
+		current->ops = ops;
+
+		DLIST_ADD(module, current);
+	}
+	*out = module;
+	return LDB_SUCCESS;
+}
+
+/*
+  initialise a chain of modules
+ */
+int ldb_module_init_chain(struct ldb_context *ldb, struct ldb_module *module)
+{
+	while (module && module->ops->init_context == NULL)
+		module = module->next;
+
+	/* init is different in that it is not an error if modules
+	 * do not require initialization */
+
+	if (module) {
+		int ret = module->ops->init_context(module);
+		if (ret != LDB_SUCCESS) {
+			ldb_debug(ldb, LDB_DEBUG_FATAL, "module %s initialization failed : %s",
+				  module->ops->name, ldb_strerror(ret));
+			return ret;
+		}
+	}
+
+	return LDB_SUCCESS;
+}
+
+int ldb_load_modules(struct ldb_context *ldb, const char *options[])
+{
+	const char *modules_string;
+	const char **modules = NULL;
+	int ret;
+	TALLOC_CTX *mem_ctx = talloc_new(ldb);
+	if (!mem_ctx) {
+		return ldb_oom(ldb);
+	}
+
+	/* find out which modules we are requested to activate */
+
+	/* check if we have a custom module list passd as ldb option */
+	if (options) {
+		modules_string = ldb_options_find(ldb, options, "modules");
+		if (modules_string) {
+			modules = ldb_modules_list_from_string(ldb, mem_ctx, modules_string);
+		}
+	}
+
+	/* if not overloaded by options and the backend is not ldap try to load the modules list from ldb */
+	if ((modules == NULL) && (strcmp("ldap", ldb->modules->ops->name) != 0)) {
+		const char * const attrs[] = { "@LIST" , NULL};
+		struct ldb_result *res = NULL;
+		struct ldb_dn *mods_dn;
+
+		mods_dn = ldb_dn_new(mem_ctx, ldb, "@MODULES");
+		if (mods_dn == NULL) {
+			talloc_free(mem_ctx);
+			return ldb_oom(ldb);
+		}
+
+		ret = ldb_search(ldb, mods_dn, &res, mods_dn, LDB_SCOPE_BASE, attrs, "@LIST=*");
+
+		if (ret == LDB_ERR_NO_SUCH_OBJECT) {
+			ldb_debug(ldb, LDB_DEBUG_TRACE, "no modules required by the db");
+		} else if (ret != LDB_SUCCESS) {
+			ldb_debug(ldb, LDB_DEBUG_FATAL, "ldb error (%s) occurred searching for modules, bailing out", ldb_errstring(ldb));
+			talloc_free(mem_ctx);
+			return ret;
+		} else {
+			const char *module_list;
+			if (res->count == 0) {
+				ldb_debug(ldb, LDB_DEBUG_TRACE, "no modules required by the db");
+			} else if (res->count > 1) {
+				ldb_debug(ldb, LDB_DEBUG_FATAL, "Too many records found (%u), bailing out", res->count);
+				talloc_free(mem_ctx);
+				return LDB_ERR_OPERATIONS_ERROR;
+			} else {
+				module_list = ldb_msg_find_attr_as_string(res->msgs[0], "@LIST", NULL);
+				if (!module_list) {
+					ldb_debug(ldb, LDB_DEBUG_TRACE, "no modules required by the db");
+				}
+				modules = ldb_modules_list_from_string(ldb, mem_ctx,
+							       module_list);
+			}
+		}
+
+		talloc_free(mods_dn);
+	}
+
+	if (modules != NULL) {
+		ret = ldb_module_load_list(ldb, modules, ldb->modules, &ldb->modules);
+		if (ret != LDB_SUCCESS) {
+			talloc_free(mem_ctx);
+			return ret;
+		}
+	} else {
+		ldb_debug(ldb, LDB_DEBUG_TRACE, "No modules specified for this database");
+	}
+
+	ret = ldb_module_init_chain(ldb, ldb->modules);
+	talloc_free(mem_ctx);
+	return ret;
+}
+
+/*
+  by using this we allow ldb modules to only implement the functions they care about,
+  which makes writing a module simpler, and makes it more likely to keep working
+  when ldb is extended
+*/
+#define FIND_OP_NOERR(module, op) do { \
+	module = module->next; \
+	while (module && module->ops->op == NULL) module = module->next; \
+	if ((module && module->ldb->flags & LDB_FLG_ENABLE_TRACING)) { \
+		ldb_debug(module->ldb, LDB_DEBUG_TRACE, "ldb_trace_next_request: (%s)->" #op, \
+			  module->ops->name);				\
+	}								\
+} while (0)
+
+#define FIND_OP(module, op) do { \
+	struct ldb_context *ldb = module->ldb; \
+	FIND_OP_NOERR(module, op); \
+	if (module == NULL) { \
+		ldb_asprintf_errstring(ldb, "Unable to find backend operation for " #op ); \
+		return LDB_ERR_OPERATIONS_ERROR;	\
+	}						\
+} while (0)
+
+
+struct ldb_module *ldb_module_new(TALLOC_CTX *memctx,
+				  struct ldb_context *ldb,
+				  const char *module_name,
+				  const struct ldb_module_ops *ops)
+{
+	struct ldb_module *module;
+
+	module = talloc(memctx, struct ldb_module);
+	if (!module) {
+		ldb_oom(ldb);
+		return NULL;
+	}
+	talloc_set_name_const(module, module_name);
+	module->ldb = ldb;
+	module->prev = module->next = NULL;
+	module->ops = ops;
+
+	return module;
+}
+
+const char * ldb_module_get_name(struct ldb_module *module)
+{
+	return module->ops->name;
+}
+
+struct ldb_context *ldb_module_get_ctx(struct ldb_module *module)
+{
+	return module->ldb;
+}
+
+const struct ldb_module_ops *ldb_module_get_ops(struct ldb_module *module)
+{
+	return module->ops;
+}
+
+void *ldb_module_get_private(struct ldb_module *module)
+{
+	return module->private_data;
+}
+
+void ldb_module_set_private(struct ldb_module *module, void *private_data)
+{
+	module->private_data = private_data;
+}
+
+/*
+   helper functions to call the next module in chain
+*/
+
+int ldb_next_request(struct ldb_module *module, struct ldb_request *request)
+{
+	int ret;
+
+	if (request->callback == NULL) {
+		ldb_set_errstring(module->ldb, "Requests MUST define callbacks");
+		return LDB_ERR_UNWILLING_TO_PERFORM;
+	}
+
+	request->handle->nesting++;
+
+	switch (request->operation) {
+	case LDB_SEARCH:
+		FIND_OP(module, search);
+		ret = module->ops->search(module, request);
+		break;
+	case LDB_ADD:
+		FIND_OP(module, add);
+		ret = module->ops->add(module, request);
+		break;
+	case LDB_MODIFY:
+		FIND_OP(module, modify);
+		ret = module->ops->modify(module, request);
+		break;
+	case LDB_DELETE:
+		FIND_OP(module, del);
+		ret = module->ops->del(module, request);
+		break;
+	case LDB_RENAME:
+		FIND_OP(module, rename);
+		ret = module->ops->rename(module, request);
+		break;
+	case LDB_EXTENDED:
+		FIND_OP(module, extended);
+		ret = module->ops->extended(module, request);
+		break;
+	default:
+		FIND_OP(module, request);
+		ret = module->ops->request(module, request);
+		break;
+	}
+
+	request->handle->nesting--;
+
+	if (ret == LDB_SUCCESS) {
+		return ret;
+	}
+	if (!ldb_errstring(module->ldb)) {
+		const char *op;
+		switch (request->operation) {
+		case LDB_SEARCH:
+			op = "LDB_SEARCH";
+			break;
+		case LDB_ADD:
+			op = "LDB_ADD";
+			break;
+		case LDB_MODIFY:
+			op = "LDB_MODIFY";
+			break;
+		case LDB_DELETE:
+			op = "LDB_DELETE";
+			break;
+		case LDB_RENAME:
+			op = "LDB_RENAME";
+			break;
+		case LDB_EXTENDED:
+			op = "LDB_EXTENDED";
+			break;
+		default:
+			op = "request";
+			break;
+		}
+
+		/* Set a default error string, to place the blame somewhere */
+		ldb_asprintf_errstring(module->ldb, "error in module %s: %s during %s (%d)", module->ops->name, ldb_strerror(ret), op, ret);
+	}
+
+	if (!(request->handle->flags & LDB_HANDLE_FLAG_DONE_CALLED)) {
+		/* It is _extremely_ common that a module returns a
+		 * failure without calling ldb_module_done(), but that
+		 * guarantees we will end up hanging in
+		 * ldb_wait(). This fixes it without having to rewrite
+		 * all our modules, and leaves us one less sharp
+		 * corner for module developers to cut themselves on
+		 */
+		ret = ldb_module_done(request, NULL, NULL, ret);
+	}
+	return ret;
+}
+
+int ldb_next_init(struct ldb_module *module)
+{
+	module = module->next;
+
+	return ldb_module_init_chain(module->ldb, module);
+}
+
+int ldb_next_start_trans(struct ldb_module *module)
+{
+	int ret;
+	FIND_OP(module, start_transaction);
+	ret = module->ops->start_transaction(module);
+	if (ret == LDB_SUCCESS) {
+		return ret;
+	}
+	if (!ldb_errstring(module->ldb)) {
+		/* Set a default error string, to place the blame somewhere */
+		ldb_asprintf_errstring(module->ldb, "start_trans error in module %s: %s (%d)", module->ops->name, ldb_strerror(ret), ret);
+	}
+	if ((module && module->ldb->flags & LDB_FLG_ENABLE_TRACING)) { 
+		ldb_debug(module->ldb, LDB_DEBUG_TRACE, "ldb_next_start_trans error: %s", 
+			  ldb_errstring(module->ldb));				
+	}
+	return ret;
+}
+
+int ldb_next_end_trans(struct ldb_module *module)
+{
+	int ret;
+	FIND_OP(module, end_transaction);
+	ret = module->ops->end_transaction(module);
+	if (ret == LDB_SUCCESS) {
+		return ret;
+	}
+	if (!ldb_errstring(module->ldb)) {
+		/* Set a default error string, to place the blame somewhere */
+		ldb_asprintf_errstring(module->ldb, "end_trans error in module %s: %s (%d)", module->ops->name, ldb_strerror(ret), ret);
+	}
+	if ((module && module->ldb->flags & LDB_FLG_ENABLE_TRACING)) { 
+		ldb_debug(module->ldb, LDB_DEBUG_TRACE, "ldb_next_end_trans error: %s", 
+			  ldb_errstring(module->ldb));				
+	}
+	return ret;
+}
+
+int ldb_next_read_lock(struct ldb_module *module)
+{
+	int ret;
+	FIND_OP(module, read_lock);
+	ret = module->ops->read_lock(module);
+	if (ret == LDB_SUCCESS) {
+		return ret;
+	}
+	if (!ldb_errstring(module->ldb)) {
+		/* Set a default error string, to place the blame somewhere */
+		ldb_asprintf_errstring(module->ldb,
+				       "read_lock error in module %s: %s (%d)",
+				       module->ops->name, ldb_strerror(ret),
+				       ret);
+	}
+	if ((module && module->ldb->flags & LDB_FLG_ENABLE_TRACING)) {
+		ldb_debug(module->ldb, LDB_DEBUG_TRACE,
+			  "ldb_next_read_lock error: %s",
+			  ldb_errstring(module->ldb));
+	}
+	return ret;
+}
+
+int ldb_next_read_unlock(struct ldb_module *module)
+{
+	int ret;
+	FIND_OP(module, read_unlock);
+	ret = module->ops->read_unlock(module);
+	if (ret == LDB_SUCCESS) {
+		return ret;
+	}
+	if (!ldb_errstring(module->ldb)) {
+		/* Set a default error string, to place the blame somewhere */
+		ldb_asprintf_errstring(module->ldb,
+				       "read_unlock error in module %s: %s (%d)",
+				       module->ops->name, ldb_strerror(ret),
+				       ret);
+	}
+	if ((module && module->ldb->flags & LDB_FLG_ENABLE_TRACING)) {
+		ldb_debug(module->ldb, LDB_DEBUG_TRACE,
+			  "ldb_next_read_unlock error: %s",
+			  ldb_errstring(module->ldb));
+	}
+	return ret;
+}
+
+int ldb_next_prepare_commit(struct ldb_module *module)
+{
+	int ret;
+	FIND_OP_NOERR(module, prepare_commit);
+	if (module == NULL) {
+		/* we are allowed to have no prepare commit in
+		   backends */
+		return LDB_SUCCESS;
+	}
+	ret = module->ops->prepare_commit(module);
+	if (ret == LDB_SUCCESS) {
+		return ret;
+	}
+	if (!ldb_errstring(module->ldb)) {
+		/* Set a default error string, to place the blame somewhere */
+		ldb_asprintf_errstring(module->ldb, "prepare_commit error in module %s: %s (%d)", module->ops->name, ldb_strerror(ret), ret);
+	}
+	if ((module && module->ldb->flags & LDB_FLG_ENABLE_TRACING)) { 
+		ldb_debug(module->ldb, LDB_DEBUG_TRACE, "ldb_next_prepare_commit error: %s", 
+			  ldb_errstring(module->ldb));				
+	}
+	return ret;
+}
+
+int ldb_next_del_trans(struct ldb_module *module)
+{
+	int ret;
+	FIND_OP(module, del_transaction);
+	ret = module->ops->del_transaction(module);
+	if (ret == LDB_SUCCESS) {
+		return ret;
+	}
+	if (!ldb_errstring(module->ldb)) {
+		/* Set a default error string, to place the blame somewhere */
+		ldb_asprintf_errstring(module->ldb, "del_trans error in module %s: %s (%d)", module->ops->name, ldb_strerror(ret), ret);
+	}
+	if ((module && module->ldb->flags & LDB_FLG_ENABLE_TRACING)) { 
+		ldb_debug(module->ldb, LDB_DEBUG_TRACE, "ldb_next_del_trans error: %s", 
+			  ldb_errstring(module->ldb));				
+	}
+	return ret;
+}
+
+/* calls the request callback to send an entry
+ *
+ * params:
+ *      req: the original request passed to your module
+ *      msg: reply message (must be a talloc pointer, and it will be stolen
+ *           on the ldb_reply that is sent to the callback)
+ * 	ctrls: controls to send in the reply  (must be a talloc pointer, and it will be stolen
+ *           on the ldb_reply that is sent to the callback)
+ */
+
+int ldb_module_send_entry(struct ldb_request *req,
+			  struct ldb_message *msg,
+			  struct ldb_control **ctrls)
+{
+	struct ldb_reply *ares;
+
+	ares = talloc_zero(req, struct ldb_reply);
+	if (!ares) {
+		ldb_oom(req->handle->ldb);
+		req->callback(req, NULL);
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	ares->type = LDB_REPLY_ENTRY;
+	ares->message = talloc_steal(ares, msg);
+	ares->controls = talloc_steal(ares, ctrls);
+	ares->error = LDB_SUCCESS;
+
+	if ((req->handle->ldb->flags & LDB_FLG_ENABLE_TRACING) &&
+	    req->handle->nesting == 0) {
+		char *s;
+		struct ldb_ldif ldif;
+		
+		ldif.changetype = LDB_CHANGETYPE_NONE;
+		ldif.msg = discard_const_p(struct ldb_message, msg);
+
+		ldb_debug_add(req->handle->ldb, "ldb_trace_response: ENTRY\n");
+
+		/* 
+		 * The choice to call
+		 * ldb_ldif_write_redacted_trace_string() is CRITICAL
+		 * for security.  It ensures that we do not output
+		 * passwords into debug logs 
+		 */
+
+		s = ldb_ldif_write_redacted_trace_string(req->handle->ldb, msg, &ldif);
+		ldb_debug_add(req->handle->ldb, "%s\n", s);
+		talloc_free(s);
+		ldb_debug_end(req->handle->ldb, LDB_DEBUG_TRACE);
+	}
+
+	return req->callback(req, ares);
+}
+
+/* calls the request callback to send an referrals
+ *
+ * params:
+ *      req: the original request passed to your module
+ *      ref: referral string (must be a talloc pointer, steal)
+ */
+
+int ldb_module_send_referral(struct ldb_request *req,
+					   char *ref)
+{
+	struct ldb_reply *ares;
+
+	ares = talloc_zero(req, struct ldb_reply);
+	if (!ares) {
+		ldb_oom(req->handle->ldb);
+		req->callback(req, NULL);
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	ares->type = LDB_REPLY_REFERRAL;
+	ares->referral = talloc_steal(ares, ref);
+	ares->error = LDB_SUCCESS;
+
+	if ((req->handle->ldb->flags & LDB_FLG_ENABLE_TRACING) &&
+	    req->handle->nesting == 0) {
+		ldb_debug_add(req->handle->ldb, "ldb_trace_response: REFERRAL\n");
+		ldb_debug_add(req->handle->ldb, "ref: %s\n", ref);
+		ldb_debug_end(req->handle->ldb, LDB_DEBUG_TRACE);
+	}
+
+	return req->callback(req, ares);
+}
+
+/* calls the original request callback
+ *
+ * params:
+ * 	req:   the original request passed to your module
+ * 	ctrls: controls to send in the reply (must be a talloc pointer, steal)
+ * 	response: results for extended request (steal)
+ * 	error: LDB_SUCCESS for a successful return
+ * 	       any other ldb error otherwise
+ */
+int ldb_module_done(struct ldb_request *req,
+		    struct ldb_control **ctrls,
+		    struct ldb_extended *response,
+		    int error)
+{
+	struct ldb_reply *ares;
+
+	ares = talloc_zero(req, struct ldb_reply);
+	if (!ares) {
+		ldb_oom(req->handle->ldb);
+		req->callback(req, NULL);
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	ares->type = LDB_REPLY_DONE;
+	ares->controls = talloc_steal(ares, ctrls);
+	ares->response = talloc_steal(ares, response);
+	ares->error = error;
+
+	req->handle->flags |= LDB_HANDLE_FLAG_DONE_CALLED;
+
+	if ((req->handle->ldb->flags & LDB_FLG_ENABLE_TRACING) &&
+	    req->handle->nesting == 0) {
+		ldb_debug_add(req->handle->ldb, "ldb_trace_response: DONE\n");
+		ldb_debug_add(req->handle->ldb, "error: %d\n", error);
+		if (ldb_errstring(req->handle->ldb)) {
+			ldb_debug_add(req->handle->ldb, "msg: %s\n",
+				  ldb_errstring(req->handle->ldb));
+		}
+		ldb_debug_end(req->handle->ldb, LDB_DEBUG_TRACE);
+	}
+
+	return req->callback(req, ares);
+}
+
+/* to be used *only* in modules init functions.
+ * this function is synchronous and will register
+ * the requested OID in the rootdse module if present
+ * otherwise it will return an error */
+int ldb_mod_register_control(struct ldb_module *module, const char *oid)
+{
+	struct ldb_request *req;
+	int ret;
+
+	req = talloc_zero(module, struct ldb_request);
+	if (req == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	req->operation = LDB_REQ_REGISTER_CONTROL;
+	req->op.reg_control.oid = oid;
+	req->callback = ldb_op_default_callback;
+
+	ldb_set_timeout(module->ldb, req, 0);
+
+	req->handle = ldb_handle_new(req, module->ldb);
+	if (req->handle == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	ret = ldb_request(module->ldb, req);
+	if (ret == LDB_SUCCESS) {
+		ret = ldb_wait(req->handle, LDB_WAIT_ALL);
+	}
+	talloc_free(req);
+
+	return ret;
+}
+
+static int ldb_modules_load_dir(const char *modules_dir, const char *version);
+
+
+/*
+  load one module. A static list of loaded module inode numbers is
+  used to prevent a module being loaded twice
+
+  dlopen() is used on the module, and dlsym() is then used to look for
+  a ldb_init_module() function. If present, that function is called
+  with the ldb version number as an argument.
+
+  The ldb_init_module() function will typically call
+  ldb_register_module() and ldb_register_backend() to register a
+  module or backend, but it may also be used to register command line
+  handling functions, ldif handlers or any other local
+  modififications.
+
+  The ldb_init_module() function does not get a ldb_context passed in,
+  as modules will be used for multiple ldb context handles. The call
+  from the first ldb_init() is just a convenient way to ensure it is
+  called early enough.
+ */
+static int ldb_modules_load_path(const char *path, const char *version)
+{
+	void *handle;
+	int (*init_fn)(const char *);
+	int ret;
+	struct stat st;
+	static struct loaded {
+		struct loaded *next, *prev;
+		ino_t st_ino;
+		dev_t st_dev;
+	} *loaded;
+	struct loaded *le;
+	int dlopen_flags;
+
+#ifdef RTLD_DEEPBIND
+	bool deepbind_enabled = (getenv("LDB_MODULES_DISABLE_DEEPBIND") == NULL);
+#endif
+
+	ret = stat(path, &st);
+	if (ret != 0) {
+		fprintf(stderr, "ldb: unable to stat module %s : %s\n", path, strerror(errno));
+		return LDB_ERR_UNAVAILABLE;
+	}
+
+	for (le=loaded; le; le=le->next) {
+		if (le->st_ino == st.st_ino &&
+		    le->st_dev == st.st_dev) {
+			/* its already loaded */
+			return LDB_SUCCESS;
+		}
+	}
+
+	le = talloc(loaded, struct loaded);
+	if (le == NULL) {
+		fprintf(stderr, "ldb: unable to allocated loaded entry\n");
+		return LDB_ERR_UNAVAILABLE;
+	}
+
+	le->st_ino = st.st_ino;
+	le->st_dev = st.st_dev;
+
+	DLIST_ADD_END(loaded, le);
+
+	/* if it is a directory, recurse */
+	if (S_ISDIR(st.st_mode)) {
+		return ldb_modules_load_dir(path, version);
+	}
+
+	dlopen_flags = RTLD_NOW;
+#ifdef RTLD_DEEPBIND
+	/*
+	 * use deepbind if possible, to avoid issues with different
+	 * system library variants, for example ldb modules may be linked
+	 * against Heimdal while the application may use MIT kerberos.
+	 *
+	 * See the dlopen manpage for details.
+	 *
+	 * One typical user is the bind_dlz module of Samba,
+	 * but symbol versioning might be enough...
+	 *
+	 * We need a way to disable this in order to allow the
+	 * ldb_*ldap modules to work with a preloaded socket wrapper.
+	 *
+	 * So in future we may remove this completely
+	 * or at least invert the default behavior.
+	*/
+	if (deepbind_enabled) {
+		dlopen_flags |= RTLD_DEEPBIND;
+	}
+#endif
+
+	handle = dlopen(path, dlopen_flags);
+	if (handle == NULL) {
+		fprintf(stderr, "ldb: unable to dlopen %s : %s\n", path, dlerror());
+		return LDB_SUCCESS;
+	}
+
+	init_fn = dlsym(handle, "ldb_init_module");
+	if (init_fn == NULL) {
+		/* ignore it, it could be an old-style
+		 * module. Once we've converted all modules we
+		 * could consider this an error */
+		dlclose(handle);
+		return LDB_SUCCESS;
+	}
+
+	ret = init_fn(version);
+	if (ret == LDB_ERR_ENTRY_ALREADY_EXISTS) {
+		/* the module is already registered - ignore this, as
+		 * it can happen if LDB_MODULES_PATH points at both
+		 * the build and install directory
+		 */
+		ret = LDB_SUCCESS;
+	}
+	return ret;
+}
+
+static int qsort_string(const char **s1, const char **s2)
+{
+	return strcmp(*s1, *s2);
+}
+
+
+/*
+  load all modules from the given ldb modules directory. This is run once
+  during the first ldb_init() call.
+
+  Modules are loaded in alphabetical order to ensure that any module
+  load ordering dependencies are reproducible. Modules should avoid
+  relying on load order
+ */
+static int ldb_modules_load_dir(const char *modules_dir, const char *version)
+{
+	DIR *dir;
+	struct dirent *de;
+	const char **modlist = NULL;
+	TALLOC_CTX *tmp_ctx = talloc_new(NULL);
+	unsigned i, num_modules = 0;
+
+	dir = opendir(modules_dir);
+	if (dir == NULL) {
+		if (errno == ENOENT) {
+			talloc_free(tmp_ctx);
+			/* we don't have any modules */
+			return LDB_SUCCESS;
+		}
+		talloc_free(tmp_ctx);
+		fprintf(stderr, "ldb: unable to open modules directory '%s' - %s\n",
+			modules_dir, strerror(errno));
+		return LDB_ERR_UNAVAILABLE;
+	}
+
+
+	while ((de = readdir(dir))) {
+		if (ISDOT(de->d_name) || ISDOTDOT(de->d_name))
+			continue;
+
+		modlist = talloc_realloc(tmp_ctx, modlist, const char *, num_modules+1);
+		if (modlist == NULL) {
+			talloc_free(tmp_ctx);
+			closedir(dir);
+			fprintf(stderr, "ldb: unable to allocate modules list\n");
+			return LDB_ERR_UNAVAILABLE;
+		}
+		modlist[num_modules] = talloc_asprintf(modlist, "%s/%s", modules_dir, de->d_name);
+		if (modlist[num_modules] == NULL) {
+			talloc_free(tmp_ctx);
+			closedir(dir);
+			fprintf(stderr, "ldb: unable to allocate module list entry\n");
+			return LDB_ERR_UNAVAILABLE;
+		}
+		num_modules++;
+	}
+
+	closedir(dir);
+
+	/* sort the directory, so we get consistent load ordering */
+	TYPESAFE_QSORT(modlist, num_modules, qsort_string);
+
+	for (i=0; i<num_modules; i++) {
+		int ret = ldb_modules_load_path(modlist[i], version);
+		if (ret != LDB_SUCCESS) {
+			fprintf(stderr, "ldb: failed to initialise module %s : %s\n",
+				modlist[i], ldb_strerror(ret));
+			talloc_free(tmp_ctx);
+			return ret;
+		}
+	}
+
+	talloc_free(tmp_ctx);
+
+	return LDB_SUCCESS;
+}
+
+/* 
+   load any additional modules from the given directory 
+*/
+void ldb_set_modules_dir(struct ldb_context *ldb, const char *path)
+{
+	int ret = ldb_modules_load_dir(path, LDB_VERSION);
+	if (ret != LDB_SUCCESS) {
+		ldb_asprintf_errstring(ldb, "Failed to load modules from: %s\n", path);
+	}
+}
+
+
+/*
+  load all modules static (builtin) modules
+ */
+static int ldb_modules_load_static(const char *version)
+{
+	static bool initialised;
+#define _MODULE_PROTO(init) extern int init(const char *);
+	STATIC_ldb_MODULES_PROTO;
+	const ldb_module_init_fn static_init_functions[] = { STATIC_ldb_MODULES };
+	unsigned i;
+
+	if (initialised) {
+		return LDB_SUCCESS;
+	}
+	initialised = true;
+
+	for (i=0; static_init_functions[i]; i++) {
+		int ret = static_init_functions[i](version);
+		if (ret != LDB_SUCCESS) {
+			return ret;
+		}
+	}
+	return LDB_SUCCESS;
+}
+
+/*
+  load all modules from the given ldb modules path, colon
+  separated.
+
+  modules are loaded recursively for all subdirectories in the paths
+ */
+int ldb_modules_load(const char *modules_path, const char *version)
+{
+	char *tok, *path, *tok_ptr=NULL;
+	int ret;
+
+	ret = ldb_modules_load_static(version);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	path = talloc_strdup(NULL, modules_path);
+	if (path == NULL) {
+		fprintf(stderr, "ldb: failed to allocate modules_path\n");
+		return LDB_ERR_UNAVAILABLE;
+	}
+
+	for (tok=strtok_r(path, ":", &tok_ptr);
+	     tok;
+	     tok=strtok_r(NULL, ":", &tok_ptr)) {
+		ret = ldb_modules_load_path(tok, version);
+		if (ret != LDB_SUCCESS) {
+			talloc_free(path);
+			return ret;
+		}
+	}
+	talloc_free(path);
+
+	return LDB_SUCCESS;
+}
+
+
+/*
+  return a string representation of the calling chain for the given
+  ldb request
+ */
+char *ldb_module_call_chain(struct ldb_request *req, TALLOC_CTX *mem_ctx)
+{
+	char *ret;
+	unsigned int i = 0;
+
+	ret = talloc_strdup(mem_ctx, "");
+	if (ret == NULL) {
+		return NULL;
+	}
+
+	while (req && req->handle) {
+		char *s = talloc_asprintf_append_buffer(ret, "req[%u] %p  : %s\n",
+							i++, req, ldb_req_location(req));
+		if (s == NULL) {
+			talloc_free(ret);
+			return NULL;
+		}
+		ret = s;
+		req = req->handle->parent;
+	}
+	return ret;
+}
+
+
+/*
+  return the next module in the chain
+ */
+struct ldb_module *ldb_module_next(struct ldb_module *module)
+{
+	return module->next;
+}
+
+/*
+  set the next module in the module chain
+ */
+void ldb_module_set_next(struct ldb_module *module, struct ldb_module *next)
+{
+	module->next = next;
+}
+
+
+/*
+  get the popt_options pointer in the ldb structure. This allows a ldb
+  module to change the command line parsing
+ */
+struct poptOption **ldb_module_popt_options(struct ldb_context *ldb)
+{
+	return &ldb->popt_options;
+}
+
+
+/*
+  return the current ldb flags LDB_FLG_*
+ */
+uint32_t ldb_module_flags(struct ldb_context *ldb)
+{
+	return ldb->flags;
+}
diff --git a/lib/ldb/common/ldb_msg.c b/lib/ldb/common/ldb_msg.c
new file mode 100644
index 0000000..2ea2cce
--- /dev/null
+++ b/lib/ldb/common/ldb_msg.c
@@ -0,0 +1,1738 @@
+/*
+   ldb database library
+
+   Copyright (C) Andrew Tridgell  2004
+
+     ** NOTE! The following LGPL license applies to the ldb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+/*
+ *  Name: ldb
+ *
+ *  Component: ldb message component utility functions
+ *
+ *  Description: functions for manipulating ldb_message structures
+ *
+ *  Author: Andrew Tridgell
+ */
+
+#include "ldb_private.h"
+
+/*
+  create a new ldb_message in a given memory context (NULL for top level)
+*/
+struct ldb_message *ldb_msg_new(TALLOC_CTX *mem_ctx)
+{
+	return talloc_zero(mem_ctx, struct ldb_message);
+}
+
+/*
+  find an element in a message by attribute name
+*/
+struct ldb_message_element *ldb_msg_find_element(const struct ldb_message *msg,
+						 const char *attr_name)
+{
+	unsigned int i;
+	for (i=0;i<msg->num_elements;i++) {
+		if (ldb_attr_cmp(msg->elements[i].name, attr_name) == 0) {
+			return &msg->elements[i];
+		}
+	}
+	return NULL;
+}
+
+/*
+  see if two ldb_val structures contain exactly the same data
+  return 1 for a match, 0 for a mis-match
+*/
+int ldb_val_equal_exact(const struct ldb_val *v1, const struct ldb_val *v2)
+{
+	if (v1->length != v2->length) return 0;
+	if (v1->data == v2->data) return 1;
+	if (v1->length == 0) return 1;
+
+	if (memcmp(v1->data, v2->data, v1->length) == 0) {
+		return 1;
+	}
+
+	return 0;
+}
+
+/*
+  find a value in an element
+  assumes case sensitive comparison
+*/
+struct ldb_val *ldb_msg_find_val(const struct ldb_message_element *el,
+				 struct ldb_val *val)
+{
+	unsigned int i;
+	for (i=0;i<el->num_values;i++) {
+		if (ldb_val_equal_exact(val, &el->values[i])) {
+			return &el->values[i];
+		}
+	}
+	return NULL;
+}
+
+
+static int ldb_val_cmp(const struct ldb_val *v1, const struct ldb_val *v2)
+{
+	if (v1->length != v2->length) {
+		return v1->length - v2->length;
+	}
+	return memcmp(v1->data, v2->data, v1->length);
+}
+
+
+/*
+  ldb_msg_find_duplicate_val() will set the **duplicate pointer to the first
+  duplicate value it finds. It does a case sensitive comparison (memcmp).
+
+  LDB_ERR_OPERATIONS_ERROR indicates an allocation failure or an unknown
+  options flag, otherwise LDB_SUCCESS.
+*/
+#define LDB_DUP_QUADRATIC_THRESHOLD 10
+
+int ldb_msg_find_duplicate_val(struct ldb_context *ldb,
+			       TALLOC_CTX *mem_ctx,
+			       const struct ldb_message_element *el,
+			       struct ldb_val **duplicate,
+			       uint32_t options)
+{
+	unsigned int i, j;
+	struct ldb_val *val;
+
+	if (options != 0) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	*duplicate = NULL;
+
+	/*
+	   If there are not many values, it is best to avoid the talloc
+	   overhead and just do a brute force search.
+	 */
+	if (el->num_values < LDB_DUP_QUADRATIC_THRESHOLD) {
+		for (j = 0; j < el->num_values; j++) {
+			val = &el->values[j];
+			for ( i = j + 1; i < el->num_values; i++) {
+				if (ldb_val_equal_exact(val, &el->values[i])) {
+					*duplicate = val;
+					return LDB_SUCCESS;
+				}
+			}
+		}
+	} else {
+		struct ldb_val *values;
+		values = talloc_array(mem_ctx, struct ldb_val, el->num_values);
+		if (values == NULL) {
+			return LDB_ERR_OPERATIONS_ERROR;
+		}
+
+		memcpy(values, el->values,
+		       el->num_values * sizeof(struct ldb_val));
+		TYPESAFE_QSORT(values, el->num_values, ldb_val_cmp);
+		for (i = 1; i < el->num_values; i++) {
+			if (ldb_val_equal_exact(&values[i],
+						&values[i - 1])) {
+				/* find the original location */
+				for (j = 0; j < el->num_values; j++) {
+					if (ldb_val_equal_exact(&values[i],
+								&el->values[j])
+						) {
+						*duplicate = &el->values[j];
+						break;
+					}
+				}
+				talloc_free(values);
+				if (*duplicate == NULL) {
+					/* how we got here, I don't know */
+					return LDB_ERR_OPERATIONS_ERROR;
+				}
+				return LDB_SUCCESS;
+			}
+		}
+		talloc_free(values);
+	}
+	return LDB_SUCCESS;
+}
+
+
+/*
+  Determine whether the values in an element are also in another element.
+
+  Without any flags, return LDB_ERR_ATTRIBUTE_OR_VALUE_EXISTS if the elements
+  share values, or LDB_SUCCESS if they don't. In this case, the function
+  simply determines the set intersection and it doesn't matter in which order
+  the elements are provided.
+
+  With the LDB_MSG_FIND_COMMON_REMOVE_DUPLICATES flag, any values in common are
+  removed from the first element and LDB_SUCCESS is returned.
+
+  LDB_ERR_OPERATIONS_ERROR indicates an allocation failure or an unknown option.
+  LDB_ERR_INAPPROPRIATE_MATCHING is returned if the elements differ in name.
+*/
+
+int ldb_msg_find_common_values(struct ldb_context *ldb,
+			       TALLOC_CTX *mem_ctx,
+			       struct ldb_message_element *el,
+			       struct ldb_message_element *el2,
+			       uint32_t options)
+{
+	struct ldb_val *values;
+	struct ldb_val *values2;
+	unsigned int i, j, k, n_values;
+
+	bool remove_duplicates = options & LDB_MSG_FIND_COMMON_REMOVE_DUPLICATES;
+
+	if ((options & ~LDB_MSG_FIND_COMMON_REMOVE_DUPLICATES) != 0) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	if (strcmp(el->name, el2->name) != 0) {
+		return LDB_ERR_INAPPROPRIATE_MATCHING;
+	}
+	if (el->num_values == 0 || el2->num_values == 0) {
+		return LDB_SUCCESS;
+	}
+	/*
+	   With few values, it is better to do the brute-force search than the
+	   clever search involving tallocs, memcpys, sorts, etc.
+	*/
+	if (MIN(el->num_values, el2->num_values) == 1 ||
+	    MAX(el->num_values, el2->num_values) < LDB_DUP_QUADRATIC_THRESHOLD) {
+		for (i = 0; i < el2->num_values; i++) {
+			for (j = 0; j < el->num_values; j++) {
+				if (ldb_val_equal_exact(&el->values[j],
+							&el2->values[i])) {
+					if (! remove_duplicates) {
+					    return			\
+					      LDB_ERR_ATTRIBUTE_OR_VALUE_EXISTS;
+					}
+					/*
+					  With the remove_duplicates flag, we
+					  resolve the intersection by removing
+					  the offending one from el.
+					*/
+					el->num_values--;
+					for (k = j; k < el->num_values; k++) {
+						el->values[k] = \
+							el->values[k + 1];
+					}
+					j--; /* rewind */
+				}
+			}
+		}
+		return LDB_SUCCESS;
+	}
+
+	values = talloc_array(mem_ctx, struct ldb_val, el->num_values);
+	if (values == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	values2 = talloc_array(mem_ctx, struct ldb_val,
+				    el2->num_values);
+	if (values2 == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	memcpy(values, el->values,
+	       el->num_values * sizeof(struct ldb_val));
+	memcpy(values2, el2->values,
+	       el2->num_values * sizeof(struct ldb_val));
+	TYPESAFE_QSORT(values, el->num_values, ldb_val_cmp);
+	TYPESAFE_QSORT(values2, el2->num_values, ldb_val_cmp);
+
+	/*
+	   el->n_values may diverge from the number of values in the sorted
+	   list when the remove_duplicates flag is used.
+	*/
+	n_values = el->num_values;
+	i = 0;
+	j = 0;
+	while (i != n_values && j < el2->num_values) {
+		int ret = ldb_val_cmp(&values[i], &values2[j]);
+		if (ret < 0) {
+			i++;
+		} else if (ret > 0) {
+			j++;
+		} else {
+			/* we have a collision */
+			if (! remove_duplicates) {
+				TALLOC_FREE(values);
+				TALLOC_FREE(values2);
+				return LDB_ERR_ATTRIBUTE_OR_VALUE_EXISTS;
+			}
+			/*
+			   With the remove_duplicates flag we need to find
+			   this in the original list and remove it, which is
+			   inefficient but hopefully rare.
+			*/
+			for (k = 0; k < el->num_values; k++) {
+				if (ldb_val_equal_exact(&el->values[k],
+							&values[i])) {
+					break;
+				}
+			}
+			el->num_values--;
+			for (; k < el->num_values; k++) {
+				el->values[k] = el->values[k + 1];
+			}
+			i++;
+		}
+	}
+	TALLOC_FREE(values);
+	TALLOC_FREE(values2);
+
+	return LDB_SUCCESS;
+}
+
+/*
+  duplicate a ldb_val structure
+*/
+struct ldb_val ldb_val_dup(TALLOC_CTX *mem_ctx, const struct ldb_val *v)
+{
+	struct ldb_val v2;
+	v2.length = v->length;
+	if (v->data == NULL) {
+		v2.data = NULL;
+		return v2;
+	}
+
+	/* the +1 is to cope with buggy C library routines like strndup
+	   that look one byte beyond */
+	v2.data = talloc_array(mem_ctx, uint8_t, v->length+1);
+	if (!v2.data) {
+		v2.length = 0;
+		return v2;
+	}
+
+	memcpy(v2.data, v->data, v->length);
+	((char *)v2.data)[v->length] = 0;
+	return v2;
+}
+
+/**
+ * Adds new empty element to msg->elements
+ */
+static int _ldb_msg_add_el(struct ldb_message *msg,
+			   struct ldb_message_element **return_el)
+{
+	struct ldb_message_element *els;
+
+	/*
+	 * TODO: Find out a way to assert on input parameters.
+	 * msg and return_el must be valid
+	 */
+
+	els = talloc_realloc(msg, msg->elements,
+			     struct ldb_message_element, msg->num_elements + 1);
+	if (!els) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	ZERO_STRUCT(els[msg->num_elements]);
+
+	msg->elements = els;
+	msg->num_elements++;
+
+	*return_el = &els[msg->num_elements-1];
+
+	return LDB_SUCCESS;
+}
+
+/**
+ * Add an empty element with a given name to a message
+ */
+int ldb_msg_add_empty(struct ldb_message *msg,
+		      const char *attr_name,
+		      int flags,
+		      struct ldb_message_element **return_el)
+{
+	int ret;
+	struct ldb_message_element *el;
+
+	ret = _ldb_msg_add_el(msg, &el);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	/* initialize newly added element */
+	el->flags = flags;
+	el->name = talloc_strdup(msg->elements, attr_name);
+	if (!el->name) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	if (return_el) {
+		*return_el = el;
+	}
+
+	return LDB_SUCCESS;
+}
+
+/**
+ * Adds an element to a message.
+ *
+ * NOTE: Ownership of ldb_message_element fields
+ *       is NOT transferred. Thus, if *el pointer
+ *       is invalidated for some reason, this will
+ *       corrupt *msg contents also
+ */
+int ldb_msg_add(struct ldb_message *msg,
+		const struct ldb_message_element *el,
+		int flags)
+{
+	int ret;
+	struct ldb_message_element *el_new;
+	/* We have to copy this, just in case *el is a pointer into
+	 * what ldb_msg_add_empty() is about to realloc() */
+	struct ldb_message_element el_copy = *el;
+
+	ret = _ldb_msg_add_el(msg, &el_new);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	el_new->flags      = flags;
+	el_new->name       = el_copy.name;
+	el_new->num_values = el_copy.num_values;
+	el_new->values     = el_copy.values;
+
+	return LDB_SUCCESS;
+}
+
+/*
+ * add a value to a message element
+ */
+int ldb_msg_element_add_value(TALLOC_CTX *mem_ctx,
+			      struct ldb_message_element *el,
+			      const struct ldb_val *val)
+{
+	struct ldb_val *vals;
+
+	if (el->flags & LDB_FLAG_INTERNAL_SHARED_VALUES) {
+		/*
+		 * Another message is using this message element's values array,
+		 * so we don't want to make any modifications to the original
+		 * message, or potentially invalidate its own values by calling
+		 * talloc_realloc(). Make a copy instead.
+		 */
+		el->flags &= ~LDB_FLAG_INTERNAL_SHARED_VALUES;
+
+		vals = talloc_array(mem_ctx, struct ldb_val,
+				    el->num_values + 1);
+		if (vals == NULL) {
+			return LDB_ERR_OPERATIONS_ERROR;
+		}
+
+		if (el->values != NULL) {
+			memcpy(vals, el->values, el->num_values * sizeof(struct ldb_val));
+		}
+	} else {
+		vals = talloc_realloc(mem_ctx, el->values, struct ldb_val,
+				      el->num_values + 1);
+		if (vals == NULL) {
+			return LDB_ERR_OPERATIONS_ERROR;
+		}
+	}
+	el->values = vals;
+	el->values[el->num_values] = *val;
+	el->num_values++;
+
+	return LDB_SUCCESS;
+}
+
+/*
+  add a value to a message
+*/
+int ldb_msg_add_value(struct ldb_message *msg,
+		      const char *attr_name,
+		      const struct ldb_val *val,
+		      struct ldb_message_element **return_el)
+{
+	struct ldb_message_element *el;
+	int ret;
+
+	el = ldb_msg_find_element(msg, attr_name);
+	if (!el) {
+		ret = ldb_msg_add_empty(msg, attr_name, 0, &el);
+		if (ret != LDB_SUCCESS) {
+			return ret;
+		}
+	}
+
+	ret = ldb_msg_element_add_value(msg->elements, el, val);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	if (return_el) {
+		*return_el = el;
+	}
+
+	return LDB_SUCCESS;
+}
+
+
+/*
+  add a value to a message, stealing it into the 'right' place
+*/
+int ldb_msg_add_steal_value(struct ldb_message *msg,
+			    const char *attr_name,
+			    struct ldb_val *val)
+{
+	int ret;
+	struct ldb_message_element *el;
+
+	ret = ldb_msg_add_value(msg, attr_name, val, &el);
+	if (ret == LDB_SUCCESS) {
+		talloc_steal(el->values, val->data);
+	}
+	return ret;
+}
+
+
+/*
+  add a string element to a message, specifying flags
+*/
+int ldb_msg_add_string_flags(struct ldb_message *msg,
+			     const char *attr_name, const char *str,
+			     int flags)
+{
+	struct ldb_val val;
+	int ret;
+	struct ldb_message_element *el = NULL;
+
+	val.data = discard_const_p(uint8_t, str);
+	val.length = strlen(str);
+
+	if (val.length == 0) {
+		/* allow empty strings as non-existent attributes */
+		return LDB_SUCCESS;
+	}
+
+	ret = ldb_msg_add_value(msg, attr_name, &val, &el);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	if (flags != 0) {
+		el->flags = flags;
+	}
+
+	return LDB_SUCCESS;
+}
+
+/*
+  add a string element to a message
+*/
+int ldb_msg_add_string(struct ldb_message *msg,
+		       const char *attr_name, const char *str)
+{
+	return ldb_msg_add_string_flags(msg, attr_name, str, 0);
+}
+
+/*
+  add a string element to a message, stealing it into the 'right' place
+*/
+int ldb_msg_add_steal_string(struct ldb_message *msg,
+			     const char *attr_name, char *str)
+{
+	struct ldb_val val;
+
+	val.data = (uint8_t *)str;
+	val.length = strlen(str);
+
+	if (val.length == 0) {
+		/* allow empty strings as non-existent attributes */
+		return LDB_SUCCESS;
+	}
+
+	return ldb_msg_add_steal_value(msg, attr_name, &val);
+}
+
+/*
+  add a DN element to a message
+  WARNING: this uses the linearized string from the dn, and does not
+  copy the string.
+*/
+int ldb_msg_add_linearized_dn(struct ldb_message *msg, const char *attr_name,
+			      struct ldb_dn *dn)
+{
+	char *str = ldb_dn_alloc_linearized(msg, dn);
+
+	if (str == NULL) {
+		/* we don't want to have unknown DNs added */
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	return ldb_msg_add_steal_string(msg, attr_name, str);
+}
+
+/*
+  add a printf formatted element to a message
+*/
+int ldb_msg_add_fmt(struct ldb_message *msg,
+		    const char *attr_name, const char *fmt, ...)
+{
+	struct ldb_val val;
+	va_list ap;
+	char *str;
+
+	va_start(ap, fmt);
+	str = talloc_vasprintf(msg, fmt, ap);
+	va_end(ap);
+
+	if (str == NULL) return LDB_ERR_OPERATIONS_ERROR;
+
+	val.data   = (uint8_t *)str;
+	val.length = strlen(str);
+
+	return ldb_msg_add_steal_value(msg, attr_name, &val);
+}
+
+static int ldb_msg_append_value_impl(struct ldb_message *msg,
+				     const char *attr_name,
+				     const struct ldb_val *val,
+				     int flags,
+				     struct ldb_message_element **return_el)
+{
+	struct ldb_message_element *el = NULL;
+	int ret;
+
+	ret = ldb_msg_add_empty(msg, attr_name, flags, &el);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	ret = ldb_msg_element_add_value(msg->elements, el, val);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	if (return_el != NULL) {
+		*return_el = el;
+	}
+
+	return LDB_SUCCESS;
+}
+
+/*
+  append a value to a message
+*/
+int ldb_msg_append_value(struct ldb_message *msg,
+			 const char *attr_name,
+			 const struct ldb_val *val,
+			 int flags)
+{
+	return ldb_msg_append_value_impl(msg, attr_name, val, flags, NULL);
+}
+
+/*
+  append a value to a message, stealing it into the 'right' place
+*/
+int ldb_msg_append_steal_value(struct ldb_message *msg,
+			       const char *attr_name,
+			       struct ldb_val *val,
+			       int flags)
+{
+	int ret;
+	struct ldb_message_element *el = NULL;
+
+	ret = ldb_msg_append_value_impl(msg, attr_name, val, flags, &el);
+	if (ret == LDB_SUCCESS) {
+		talloc_steal(el->values, val->data);
+	}
+	return ret;
+}
+
+/*
+  append a string element to a message, stealing it into the 'right' place
+*/
+int ldb_msg_append_steal_string(struct ldb_message *msg,
+				const char *attr_name, char *str,
+				int flags)
+{
+	struct ldb_val val;
+
+	val.data = (uint8_t *)str;
+	val.length = strlen(str);
+
+	if (val.length == 0) {
+		/* allow empty strings as non-existent attributes */
+		return LDB_SUCCESS;
+	}
+
+	return ldb_msg_append_steal_value(msg, attr_name, &val, flags);
+}
+
+/*
+  append a string element to a message
+*/
+int ldb_msg_append_string(struct ldb_message *msg,
+			  const char *attr_name, const char *str, int flags)
+{
+	struct ldb_val val;
+
+	val.data = discard_const_p(uint8_t, str);
+	val.length = strlen(str);
+
+	if (val.length == 0) {
+		/* allow empty strings as non-existent attributes */
+		return LDB_SUCCESS;
+	}
+
+	return ldb_msg_append_value(msg, attr_name, &val, flags);
+}
+
+/*
+  append a DN element to a message
+  WARNING: this uses the linearized string from the dn, and does not
+  copy the string.
+*/
+int ldb_msg_append_linearized_dn(struct ldb_message *msg, const char *attr_name,
+				 struct ldb_dn *dn, int flags)
+{
+	char *str = ldb_dn_alloc_linearized(msg, dn);
+
+	if (str == NULL) {
+		/* we don't want to have unknown DNs added */
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	return ldb_msg_append_steal_string(msg, attr_name, str, flags);
+}
+
+/*
+  append a printf formatted element to a message
+*/
+int ldb_msg_append_fmt(struct ldb_message *msg, int flags,
+		       const char *attr_name, const char *fmt, ...)
+{
+	struct ldb_val val;
+	va_list ap;
+	char *str = NULL;
+
+	va_start(ap, fmt);
+	str = talloc_vasprintf(msg, fmt, ap);
+	va_end(ap);
+
+	if (str == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	val.data   = (uint8_t *)str;
+	val.length = strlen(str);
+
+	return ldb_msg_append_steal_value(msg, attr_name, &val, flags);
+}
+
+/*
+  compare two ldb_message_element structures
+  assumes case sensitive comparison
+*/
+int ldb_msg_element_compare(struct ldb_message_element *el1,
+			    struct ldb_message_element *el2)
+{
+	unsigned int i;
+
+	if (el1->num_values != el2->num_values) {
+		return el1->num_values - el2->num_values;
+	}
+
+	for (i=0;i<el1->num_values;i++) {
+		if (!ldb_msg_find_val(el2, &el1->values[i])) {
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+/*
+  compare two ldb_message_element structures.
+  Different ordering is considered a mismatch
+*/
+bool ldb_msg_element_equal_ordered(const struct ldb_message_element *el1,
+				   const struct ldb_message_element *el2)
+{
+	unsigned i;
+	if (el1->num_values != el2->num_values) {
+		return false;
+	}
+	for (i=0;i<el1->num_values;i++) {
+		if (ldb_val_equal_exact(&el1->values[i],
+					&el2->values[i]) != 1) {
+			return false;
+		}
+	}
+	return true;
+}
+
+/*
+  compare two ldb_message_element structures
+  comparing by element name
+*/
+int ldb_msg_element_compare_name(struct ldb_message_element *el1,
+				 struct ldb_message_element *el2)
+{
+	return ldb_attr_cmp(el1->name, el2->name);
+}
+
+void ldb_msg_element_mark_inaccessible(struct ldb_message_element *el)
+{
+	el->flags |= LDB_FLAG_INTERNAL_INACCESSIBLE_ATTRIBUTE;
+}
+
+bool ldb_msg_element_is_inaccessible(const struct ldb_message_element *el)
+{
+	return (el->flags & LDB_FLAG_INTERNAL_INACCESSIBLE_ATTRIBUTE) != 0;
+}
+
+void ldb_msg_remove_inaccessible(struct ldb_message *msg)
+{
+	unsigned i;
+	unsigned num_del = 0;
+
+	for (i = 0; i < msg->num_elements; ++i) {
+		if (ldb_msg_element_is_inaccessible(&msg->elements[i])) {
+			++num_del;
+		} else if (num_del) {
+			msg->elements[i - num_del] = msg->elements[i];
+		}
+	}
+
+	msg->num_elements -= num_del;
+}
+
+/*
+  convenience functions to return common types from a message
+  these return the first value if the attribute is multi-valued
+*/
+const struct ldb_val *ldb_msg_find_ldb_val(const struct ldb_message *msg,
+					   const char *attr_name)
+{
+	struct ldb_message_element *el = ldb_msg_find_element(msg, attr_name);
+	if (!el || el->num_values == 0) {
+		return NULL;
+	}
+	return &el->values[0];
+}
+
+int ldb_msg_find_attr_as_int(const struct ldb_message *msg,
+			     const char *attr_name,
+			     int default_value)
+{
+	const struct ldb_val *v = ldb_msg_find_ldb_val(msg, attr_name);
+	char buf[sizeof("-2147483648")];
+	char *end = NULL;
+	int ret;
+
+	if (!v || !v->data) {
+		return default_value;
+	}
+
+	ZERO_STRUCT(buf);
+	if (v->length >= sizeof(buf)) {
+		return default_value;
+	}
+
+	memcpy(buf, v->data, v->length);
+	errno = 0;
+	ret = (int) strtoll(buf, &end, 10);
+	if (errno != 0) {
+		return default_value;
+	}
+	if (end && end[0] != '\0') {
+		return default_value;
+	}
+	return ret;
+}
+
+unsigned int ldb_msg_find_attr_as_uint(const struct ldb_message *msg,
+				       const char *attr_name,
+				       unsigned int default_value)
+{
+	const struct ldb_val *v = ldb_msg_find_ldb_val(msg, attr_name);
+	char buf[sizeof("-2147483648")];
+	char *end = NULL;
+	unsigned int ret;
+
+	if (!v || !v->data) {
+		return default_value;
+	}
+
+	ZERO_STRUCT(buf);
+	if (v->length >= sizeof(buf)) {
+		return default_value;
+	}
+
+	memcpy(buf, v->data, v->length);
+	errno = 0;
+	ret = (unsigned int) strtoll(buf, &end, 10);
+	if (errno != 0) {
+		errno = 0;
+		ret = (unsigned int) strtoull(buf, &end, 10);
+		if (errno != 0) {
+			return default_value;
+		}
+	}
+	if (end && end[0] != '\0') {
+		return default_value;
+	}
+	return ret;
+}
+
+int64_t ldb_msg_find_attr_as_int64(const struct ldb_message *msg,
+				   const char *attr_name,
+				   int64_t default_value)
+{
+	const struct ldb_val *v = ldb_msg_find_ldb_val(msg, attr_name);
+	char buf[sizeof("-9223372036854775808")];
+	char *end = NULL;
+	int64_t ret;
+
+	if (!v || !v->data) {
+		return default_value;
+	}
+
+	ZERO_STRUCT(buf);
+	if (v->length >= sizeof(buf)) {
+		return default_value;
+	}
+
+	memcpy(buf, v->data, v->length);
+	errno = 0;
+	ret = (int64_t) strtoll(buf, &end, 10);
+	if (errno != 0) {
+		return default_value;
+	}
+	if (end && end[0] != '\0') {
+		return default_value;
+	}
+	return ret;
+}
+
+uint64_t ldb_msg_find_attr_as_uint64(const struct ldb_message *msg,
+				     const char *attr_name,
+				     uint64_t default_value)
+{
+	const struct ldb_val *v = ldb_msg_find_ldb_val(msg, attr_name);
+	char buf[sizeof("-9223372036854775808")];
+	char *end = NULL;
+	uint64_t ret;
+
+	if (!v || !v->data) {
+		return default_value;
+	}
+
+	ZERO_STRUCT(buf);
+	if (v->length >= sizeof(buf)) {
+		return default_value;
+	}
+
+	memcpy(buf, v->data, v->length);
+	errno = 0;
+	ret = (uint64_t) strtoll(buf, &end, 10);
+	if (errno != 0) {
+		errno = 0;
+		ret = (uint64_t) strtoull(buf, &end, 10);
+		if (errno != 0) {
+			return default_value;
+		}
+	}
+	if (end && end[0] != '\0') {
+		return default_value;
+	}
+	return ret;
+}
+
+double ldb_msg_find_attr_as_double(const struct ldb_message *msg,
+				   const char *attr_name,
+				   double default_value)
+{
+	const struct ldb_val *v = ldb_msg_find_ldb_val(msg, attr_name);
+	char *buf;
+	char *end = NULL;
+	double ret;
+
+	if (!v || !v->data) {
+		return default_value;
+	}
+	buf = talloc_strndup(msg, (const char *)v->data, v->length);
+	if (buf == NULL) {
+		return default_value;
+	}
+
+	errno = 0;
+	ret = strtod(buf, &end);
+	talloc_free(buf);
+	if (errno != 0) {
+		return default_value;
+	}
+	if (end && end[0] != '\0') {
+		return default_value;
+	}
+	return ret;
+}
+
+int ldb_msg_find_attr_as_bool(const struct ldb_message *msg,
+			      const char *attr_name,
+			      int default_value)
+{
+	const struct ldb_val *v = ldb_msg_find_ldb_val(msg, attr_name);
+	if (!v || !v->data) {
+		return default_value;
+	}
+	if (v->length == 5 && strncasecmp((const char *)v->data, "FALSE", 5) == 0) {
+		return 0;
+	}
+	if (v->length == 4 && strncasecmp((const char *)v->data, "TRUE", 4) == 0) {
+		return 1;
+	}
+	return default_value;
+}
+
+const char *ldb_msg_find_attr_as_string(const struct ldb_message *msg,
+					const char *attr_name,
+					const char *default_value)
+{
+	const struct ldb_val *v = ldb_msg_find_ldb_val(msg, attr_name);
+	if (!v || !v->data) {
+		return default_value;
+	}
+	if (v->data[v->length] != '\0') {
+		return default_value;
+	}
+	return (const char *)v->data;
+}
+
+struct ldb_dn *ldb_msg_find_attr_as_dn(struct ldb_context *ldb,
+				       TALLOC_CTX *mem_ctx,
+				       const struct ldb_message *msg,
+				       const char *attr_name)
+{
+	struct ldb_dn *res_dn;
+	const struct ldb_val *v;
+
+	v = ldb_msg_find_ldb_val(msg, attr_name);
+	if (!v || !v->data) {
+		return NULL;
+	}
+	res_dn = ldb_dn_from_ldb_val(mem_ctx, ldb, v);
+	if ( ! ldb_dn_validate(res_dn)) {
+		talloc_free(res_dn);
+		return NULL;
+	}
+	return res_dn;
+}
+
+/*
+  sort the elements of a message by name
+*/
+void ldb_msg_sort_elements(struct ldb_message *msg)
+{
+	TYPESAFE_QSORT(msg->elements, msg->num_elements,
+		       ldb_msg_element_compare_name);
+}
+
+static struct ldb_message *ldb_msg_copy_shallow_impl(TALLOC_CTX *mem_ctx,
+					 const struct ldb_message *msg)
+{
+	struct ldb_message *msg2;
+	unsigned int i;
+
+	msg2 = talloc(mem_ctx, struct ldb_message);
+	if (msg2 == NULL) return NULL;
+
+	*msg2 = *msg;
+
+	msg2->elements = talloc_array(msg2, struct ldb_message_element,
+				      msg2->num_elements);
+	if (msg2->elements == NULL) goto failed;
+
+	for (i=0;i<msg2->num_elements;i++) {
+		msg2->elements[i] = msg->elements[i];
+	}
+
+	return msg2;
+
+failed:
+	talloc_free(msg2);
+	return NULL;
+}
+
+/*
+  shallow copy a message - copying only the elements array so that the caller
+  can safely add new elements without changing the message
+*/
+struct ldb_message *ldb_msg_copy_shallow(TALLOC_CTX *mem_ctx,
+					 const struct ldb_message *msg)
+{
+	struct ldb_message *msg2;
+	unsigned int i;
+
+	msg2 = ldb_msg_copy_shallow_impl(mem_ctx, msg);
+	if (msg2 == NULL) {
+		return NULL;
+	}
+
+	for (i = 0; i < msg2->num_elements; ++i) {
+		/*
+		 * Mark this message's elements as sharing their values with the
+		 * original message, so that we don't inadvertently modify or
+		 * free them. We don't mark the original message element as
+		 * shared, so the original message element should not be
+		 * modified or freed while the shallow copy lives.
+		 */
+		struct ldb_message_element *el = &msg2->elements[i];
+		el->flags |= LDB_FLAG_INTERNAL_SHARED_VALUES;
+	}
+
+        return msg2;
+}
+
+/*
+  copy a message, allocating new memory for all parts
+*/
+struct ldb_message *ldb_msg_copy(TALLOC_CTX *mem_ctx,
+				 const struct ldb_message *msg)
+{
+	struct ldb_message *msg2;
+	unsigned int i, j;
+
+	msg2 = ldb_msg_copy_shallow_impl(mem_ctx, msg);
+	if (msg2 == NULL) return NULL;
+
+	if (msg2->dn != NULL) {
+		msg2->dn = ldb_dn_copy(msg2, msg2->dn);
+		if (msg2->dn == NULL) goto failed;
+	}
+
+	for (i=0;i<msg2->num_elements;i++) {
+		struct ldb_message_element *el = &msg2->elements[i];
+		struct ldb_val *values = el->values;
+		el->name = talloc_strdup(msg2->elements, el->name);
+		if (el->name == NULL) goto failed;
+		el->values = talloc_array(msg2->elements, struct ldb_val, el->num_values);
+		if (el->values == NULL) goto failed;
+		for (j=0;j<el->num_values;j++) {
+			el->values[j] = ldb_val_dup(el->values, &values[j]);
+			if (el->values[j].data == NULL && values[j].length != 0) {
+				goto failed;
+			}
+		}
+
+                /*
+                 * Since we copied this element's values, we can mark them as
+                 * not shared.
+		 */
+		el->flags &= ~LDB_FLAG_INTERNAL_SHARED_VALUES;
+	}
+
+	return msg2;
+
+failed:
+	talloc_free(msg2);
+	return NULL;
+}
+
+
+/**
+ * Canonicalize a message, merging elements of the same name
+ */
+struct ldb_message *ldb_msg_canonicalize(struct ldb_context *ldb,
+					 const struct ldb_message *msg)
+{
+	int ret;
+	struct ldb_message *msg2;
+
+	/*
+	 * Preserve previous behavior and allocate
+	 * *msg2 into *ldb context
+	 */
+	ret = ldb_msg_normalize(ldb, ldb, msg, &msg2);
+	if (ret != LDB_SUCCESS) {
+		return NULL;
+	}
+
+	return msg2;
+}
+
+/**
+ * Canonicalize a message, merging elements of the same name
+ */
+int ldb_msg_normalize(struct ldb_context *ldb,
+		      TALLOC_CTX *mem_ctx,
+		      const struct ldb_message *msg,
+		      struct ldb_message **_msg_out)
+{
+	unsigned int i;
+	struct ldb_message *msg2;
+
+	msg2 = ldb_msg_copy(mem_ctx, msg);
+	if (msg2 == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	ldb_msg_sort_elements(msg2);
+
+	for (i=1; i < msg2->num_elements; i++) {
+		struct ldb_message_element *el1 = &msg2->elements[i-1];
+		struct ldb_message_element *el2 = &msg2->elements[i];
+
+		if (ldb_msg_element_compare_name(el1, el2) == 0) {
+			el1->values = talloc_realloc(msg2->elements,
+			                             el1->values, struct ldb_val,
+			                             el1->num_values + el2->num_values);
+			if (el1->num_values + el2->num_values > 0 && el1->values == NULL) {
+				talloc_free(msg2);
+				return LDB_ERR_OPERATIONS_ERROR;
+			}
+			memcpy(el1->values + el1->num_values,
+			       el2->values,
+			       sizeof(struct ldb_val) * el2->num_values);
+			el1->num_values += el2->num_values;
+			talloc_free(discard_const_p(char, el2->name));
+			if ((i+1) < msg2->num_elements) {
+				memmove(el2, el2+1, sizeof(struct ldb_message_element) *
+					(msg2->num_elements - (i+1)));
+			}
+			msg2->num_elements--;
+			i--;
+		}
+	}
+
+	*_msg_out = msg2;
+	return LDB_SUCCESS;
+}
+
+
+/**
+ * return a ldb_message representing the differences between msg1 and msg2.
+ * If you then use this in a ldb_modify() call,
+ * it can be used to save edits to a message
+ */
+struct ldb_message *ldb_msg_diff(struct ldb_context *ldb,
+				 struct ldb_message *msg1,
+				 struct ldb_message *msg2)
+{
+	int ldb_ret;
+	struct ldb_message *mod;
+
+	ldb_ret = ldb_msg_difference(ldb, ldb, msg1, msg2, &mod);
+	if (ldb_ret != LDB_SUCCESS) {
+		return NULL;
+	}
+
+	return mod;
+}
+
+/**
+ * return a ldb_message representing the differences between msg1 and msg2.
+ * If you then use this in a ldb_modify() call it can be used to save edits to a message
+ *
+ * Result message is constructed as follows:
+ * - LDB_FLAG_MOD_ADD     - elements found only in msg2
+ * - LDB_FLAG_MOD_REPLACE - elements in msg2 that have different value in msg1
+ *                          Value for msg2 element is used
+ * - LDB_FLAG_MOD_DELETE  - elements found only in msg2
+ *
+ * @return LDB_SUCCESS or LDB_ERR_OPERATIONS_ERROR
+ */
+int ldb_msg_difference(struct ldb_context *ldb,
+		       TALLOC_CTX *mem_ctx,
+		       struct ldb_message *msg1,
+		       struct ldb_message *msg2,
+		       struct ldb_message **_msg_out)
+{
+	int ldb_res;
+	unsigned int i;
+	struct ldb_message *mod;
+	struct ldb_message_element *el;
+	TALLOC_CTX *temp_ctx;
+
+	temp_ctx = talloc_new(mem_ctx);
+	if (!temp_ctx) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	mod = ldb_msg_new(temp_ctx);
+	if (mod == NULL) {
+		goto failed;
+	}
+
+	mod->dn = msg1->dn;
+	mod->num_elements = 0;
+	mod->elements = NULL;
+
+	/*
+	 * Canonicalize *msg2 so we have no repeated elements
+	 * Resulting message is allocated in *mod's mem context,
+	 * as we are going to move some elements from *msg2 to
+	 * *mod object later
+	 */
+	ldb_res = ldb_msg_normalize(ldb, mod, msg2, &msg2);
+	if (ldb_res != LDB_SUCCESS) {
+		goto failed;
+	}
+
+	/* look in msg2 to find elements that need to be added or modified */
+	for (i=0;i<msg2->num_elements;i++) {
+		el = ldb_msg_find_element(msg1, msg2->elements[i].name);
+
+		if (el && ldb_msg_element_compare(el, &msg2->elements[i]) == 0) {
+			continue;
+		}
+
+		ldb_res = ldb_msg_add(mod,
+		                      &msg2->elements[i],
+		                      el ? LDB_FLAG_MOD_REPLACE : LDB_FLAG_MOD_ADD);
+		if (ldb_res != LDB_SUCCESS) {
+			goto failed;
+		}
+	}
+
+	/* look in msg1 to find elements that need to be deleted */
+	for (i=0;i<msg1->num_elements;i++) {
+		el = ldb_msg_find_element(msg2, msg1->elements[i].name);
+		if (el == NULL) {
+			ldb_res = ldb_msg_add_empty(mod,
+			                            msg1->elements[i].name,
+			                            LDB_FLAG_MOD_DELETE, NULL);
+			if (ldb_res != LDB_SUCCESS) {
+				goto failed;
+			}
+		}
+	}
+
+	/* steal resulting message into supplied context */
+	talloc_steal(mem_ctx, mod);
+	*_msg_out = mod;
+
+	talloc_free(temp_ctx);
+	return LDB_SUCCESS;
+
+failed:
+	talloc_free(temp_ctx);
+	return LDB_ERR_OPERATIONS_ERROR;
+}
+
+
+int ldb_msg_sanity_check(struct ldb_context *ldb,
+			 const struct ldb_message *msg)
+{
+	unsigned int i, j;
+
+	/* basic check on DN */
+	if (msg->dn == NULL) {
+		ldb_set_errstring(ldb, "ldb message lacks a DN!");
+		return LDB_ERR_INVALID_DN_SYNTAX;
+	}
+
+	/* basic syntax checks */
+	for (i = 0; i < msg->num_elements; i++) {
+		for (j = 0; j < msg->elements[i].num_values; j++) {
+			if (msg->elements[i].values[j].length == 0) {
+				/* an attribute cannot be empty */
+				ldb_asprintf_errstring(ldb, "Element %s has empty attribute in ldb message (%s)!",
+							    msg->elements[i].name,
+							    ldb_dn_get_linearized(msg->dn));
+				return LDB_ERR_INVALID_ATTRIBUTE_SYNTAX;
+			}
+		}
+	}
+
+	return LDB_SUCCESS;
+}
+
+
+
+
+/*
+  copy an attribute list. This only copies the array, not the elements
+  (ie. the elements are left as the same pointers)
+*/
+const char **ldb_attr_list_copy(TALLOC_CTX *mem_ctx, const char * const *attrs)
+{
+	const char **ret;
+	unsigned int i;
+
+	for (i=0;attrs && attrs[i];i++) /* noop */ ;
+	ret = talloc_array(mem_ctx, const char *, i+1);
+	if (ret == NULL) {
+		return NULL;
+	}
+	for (i=0;attrs && attrs[i];i++) {
+		ret[i] = attrs[i];
+	}
+	ret[i] = attrs[i];
+	return ret;
+}
+
+
+/*
+  copy an attribute list. This only copies the array, not the elements
+  (ie. the elements are left as the same pointers).  The new attribute is added to the list.
+*/
+const char **ldb_attr_list_copy_add(TALLOC_CTX *mem_ctx, const char * const *attrs, const char *new_attr)
+{
+	const char **ret;
+	unsigned int i;
+	bool found = false;
+
+	for (i=0;attrs && attrs[i];i++) {
+		if (ldb_attr_cmp(attrs[i], new_attr) == 0) {
+			found = true;
+		}
+	}
+	if (found) {
+		return ldb_attr_list_copy(mem_ctx, attrs);
+	}
+	ret = talloc_array(mem_ctx, const char *, i+2);
+	if (ret == NULL) {
+		return NULL;
+	}
+	for (i=0;attrs && attrs[i];i++) {
+		ret[i] = attrs[i];
+	}
+	ret[i] = new_attr;
+	ret[i+1] = NULL;
+	return ret;
+}
+
+
+/*
+  return 1 if an attribute is in a list of attributes, or 0 otherwise
+*/
+int ldb_attr_in_list(const char * const *attrs, const char *attr)
+{
+	unsigned int i;
+	for (i=0;attrs && attrs[i];i++) {
+		if (ldb_attr_cmp(attrs[i], attr) == 0) {
+			return 1;
+		}
+	}
+	return 0;
+}
+
+
+/*
+  rename the specified attribute in a search result
+*/
+int ldb_msg_rename_attr(struct ldb_message *msg, const char *attr, const char *replace)
+{
+	struct ldb_message_element *el = ldb_msg_find_element(msg, attr);
+	if (el == NULL) {
+		return LDB_SUCCESS;
+	}
+	el->name = talloc_strdup(msg->elements, replace);
+	if (el->name == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	return LDB_SUCCESS;
+}
+
+
+/*
+  copy the specified attribute in a search result to a new attribute
+*/
+int ldb_msg_copy_attr(struct ldb_message *msg, const char *attr, const char *replace)
+{
+	struct ldb_message_element *el = ldb_msg_find_element(msg, attr);
+	int ret;
+
+	if (el == NULL) {
+		return LDB_SUCCESS;
+	}
+	ret = ldb_msg_add(msg, el, 0);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+	return ldb_msg_rename_attr(msg, attr, replace);
+}
+
+/*
+  remove the specified element in a search result
+*/
+void ldb_msg_remove_element(struct ldb_message *msg, struct ldb_message_element *el)
+{
+	ptrdiff_t n = (el - msg->elements);
+	if (n >= msg->num_elements || n < 0) {
+		/* the element is not in the list. the caller is crazy. */
+		return;
+	}
+	msg->num_elements--;
+	if (n != msg->num_elements) {
+		memmove(el, el+1, (msg->num_elements - n)*sizeof(*el));
+	}
+}
+
+
+/*
+  remove the specified attribute in a search result
+*/
+void ldb_msg_remove_attr(struct ldb_message *msg, const char *attr)
+{
+	struct ldb_message_element *el;
+
+	while ((el = ldb_msg_find_element(msg, attr)) != NULL) {
+		ldb_msg_remove_element(msg, el);
+	}
+}
+
+/* Reallocate elements to drop any excess capacity. */
+void ldb_msg_shrink_to_fit(struct ldb_message *msg)
+{
+	if (msg->num_elements > 0) {
+		struct ldb_message_element *elements = talloc_realloc(msg,
+								      msg->elements,
+								      struct ldb_message_element,
+								      msg->num_elements);
+		if (elements != NULL) {
+			msg->elements = elements;
+		}
+	} else {
+		TALLOC_FREE(msg->elements);
+	}
+}
+
+/*
+  return a LDAP formatted GeneralizedTime string
+*/
+char *ldb_timestring(TALLOC_CTX *mem_ctx, time_t t)
+{
+	struct tm *tm = gmtime(&t);
+	char *ts;
+	int r;
+
+	if (!tm) {
+		return NULL;
+	}
+
+	/* we now excatly how long this string will be */
+	ts = talloc_array(mem_ctx, char, 18);
+
+	/* formatted like: 20040408072012.0Z */
+	r = snprintf(ts, 18,
+			"%04u%02u%02u%02u%02u%02u.0Z",
+			tm->tm_year+1900, tm->tm_mon+1,
+			tm->tm_mday, tm->tm_hour, tm->tm_min,
+			tm->tm_sec);
+
+	if (r != 17) {
+		talloc_free(ts);
+		errno = EOVERFLOW;
+		return NULL;
+	}
+
+	return ts;
+}
+
+/*
+  convert a LDAP GeneralizedTime string to a time_t. Return 0 if unable to convert
+*/
+time_t ldb_string_to_time(const char *s)
+{
+	struct tm tm;
+
+	if (s == NULL) return 0;
+
+	memset(&tm, 0, sizeof(tm));
+	if (sscanf(s, "%04u%02u%02u%02u%02u%02u.0Z",
+		   &tm.tm_year, &tm.tm_mon, &tm.tm_mday,
+		   &tm.tm_hour, &tm.tm_min, &tm.tm_sec) != 6) {
+		return 0;
+	}
+	tm.tm_year -= 1900;
+	tm.tm_mon -= 1;
+
+	return timegm(&tm);
+}
+
+/*
+  convert a LDAP GeneralizedTime string in ldb_val format to a
+  time_t.
+*/
+int ldb_val_to_time(const struct ldb_val *v, time_t *t)
+{
+	char val[15] = {0};
+	struct tm tm = {
+		.tm_year = 0,
+	};
+
+	if (v == NULL) {
+		return LDB_ERR_INVALID_ATTRIBUTE_SYNTAX;
+	}
+
+	if (v->data == NULL) {
+		return LDB_ERR_INVALID_ATTRIBUTE_SYNTAX;
+	}
+
+	if (v->length < 16 && v->length != 13) {
+		return LDB_ERR_INVALID_ATTRIBUTE_SYNTAX;
+	}
+
+	if (v->data[v->length - 1] != 'Z') {
+		return LDB_ERR_INVALID_ATTRIBUTE_SYNTAX;
+	}
+
+	if (v->length == 13) {
+		memcpy(val, v->data, 12);
+
+		if (sscanf(val, "%02u%02u%02u%02u%02u%02u",
+			&tm.tm_year, &tm.tm_mon, &tm.tm_mday,
+			&tm.tm_hour, &tm.tm_min, &tm.tm_sec) != 6) {
+			return LDB_ERR_INVALID_ATTRIBUTE_SYNTAX;
+		}
+		if (tm.tm_year < 50) {
+			tm.tm_year += 100;
+		}
+	} else {
+
+		/*
+		 * anything between '.' and 'Z' is silently ignored.
+		 */
+		if (v->data[14] != '.') {
+			return LDB_ERR_INVALID_ATTRIBUTE_SYNTAX;
+		}
+
+		memcpy(val, v->data, 14);
+
+		if (sscanf(val, "%04u%02u%02u%02u%02u%02u",
+			&tm.tm_year, &tm.tm_mon, &tm.tm_mday,
+			&tm.tm_hour, &tm.tm_min, &tm.tm_sec) != 6) {
+			return LDB_ERR_INVALID_ATTRIBUTE_SYNTAX;
+		}
+		tm.tm_year -= 1900;
+	}
+	tm.tm_mon -= 1;
+
+	*t = timegm(&tm);
+
+	return LDB_SUCCESS;
+}
+
+/*
+  return a LDAP formatted UTCTime string
+*/
+char *ldb_timestring_utc(TALLOC_CTX *mem_ctx, time_t t)
+{
+	struct tm *tm = gmtime(&t);
+	char *ts;
+	int r;
+
+	if (!tm) {
+		return NULL;
+	}
+
+	/* we now excatly how long this string will be */
+	ts = talloc_array(mem_ctx, char, 14);
+
+	/* formatted like: 20040408072012.0Z => 040408072012Z */
+	r = snprintf(ts, 14,
+			"%02u%02u%02u%02u%02u%02uZ",
+			(tm->tm_year+1900)%100, tm->tm_mon+1,
+			tm->tm_mday, tm->tm_hour, tm->tm_min,
+			tm->tm_sec);
+
+	if (r != 13) {
+		talloc_free(ts);
+		return NULL;
+	}
+
+	return ts;
+}
+
+/*
+  convert a LDAP UTCTime string to a time_t. Return 0 if unable to convert
+*/
+time_t ldb_string_utc_to_time(const char *s)
+{
+	struct tm tm;
+
+	if (s == NULL) return 0;
+
+	memset(&tm, 0, sizeof(tm));
+	if (sscanf(s, "%02u%02u%02u%02u%02u%02uZ",
+		   &tm.tm_year, &tm.tm_mon, &tm.tm_mday,
+		   &tm.tm_hour, &tm.tm_min, &tm.tm_sec) != 6) {
+		return 0;
+	}
+	if (tm.tm_year < 50) {
+		tm.tm_year += 100;
+	}
+	tm.tm_mon -= 1;
+
+	return timegm(&tm);
+}
+
+
+/*
+  dump a set of results to a file. Useful from within gdb
+*/
+void ldb_dump_results(struct ldb_context *ldb, struct ldb_result *result, FILE *f)
+{
+	unsigned int i;
+
+	for (i = 0; i < result->count; i++) {
+		struct ldb_ldif ldif;
+		fprintf(f, "# record %d\n", i+1);
+		ldif.changetype = LDB_CHANGETYPE_NONE;
+		ldif.msg = result->msgs[i];
+		ldb_ldif_write_file(ldb, f, &ldif);
+	}
+}
+
+/*
+  checks for a string attribute. Returns "1" on match and otherwise "0".
+*/
+int ldb_msg_check_string_attribute(const struct ldb_message *msg,
+				   const char *name, const char *value)
+{
+	struct ldb_message_element *el;
+	struct ldb_val val;
+
+	el = ldb_msg_find_element(msg, name);
+	if (el == NULL) {
+		return 0;
+	}
+
+	val.data = discard_const_p(uint8_t, value);
+	val.length = strlen(value);
+
+	if (ldb_msg_find_val(el, &val)) {
+		return 1;
+	}
+
+	return 0;
+}
+
+
+/*
+  compare a ldb_val to a string
+*/
+int ldb_val_string_cmp(const struct ldb_val *v, const char *str)
+{
+	size_t len = strlen(str);
+	if (len != v->length) {
+		return len - v->length;
+	}
+	return strncmp((const char *)v->data, str, len);
+}
diff --git a/lib/ldb/common/ldb_options.c b/lib/ldb/common/ldb_options.c
new file mode 100644
index 0000000..ee8c728
--- /dev/null
+++ b/lib/ldb/common/ldb_options.c
@@ -0,0 +1,107 @@
+/*
+   ldb database library
+
+   Copyright (C) Andrew Tridgell  2010
+
+     ** NOTE! The following LGPL license applies to the ldb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+/*
+ *  Name: ldb
+ *
+ *  Component: ldb options[] handling
+ *
+ *  Author: Andrew Tridgell
+ */
+
+#include "ldb_private.h"
+
+/*
+  find an option within an options array
+
+  accepts the following forms:
+
+     NAME
+     NAME:value
+     NAME=value
+
+  returns a pointer into an element of the options[] array, or NULL is
+  not found.
+
+  For the NAME form, returns a pointer to an empty string (thus
+  allowing for boolean options).
+ */
+const char *ldb_options_find(struct ldb_context *ldb, const char *options[],
+				       const char *option_name)
+{
+	size_t len = strlen(option_name);
+	int i;
+
+	if (options == NULL) {
+		return NULL;
+	}
+
+	for (i=0; options[i]; i++) {
+		if (strncmp(option_name, options[i], len) != 0) {
+			continue;
+		}
+		if (options[i][len] == ':' || options[i][len] == '=') {
+			return &options[i][len+1];
+		}
+		if (options[i][len] == 0) {
+			return &options[i][len];
+		}
+	}
+
+	return NULL;
+}
+
+const char **ldb_options_copy(TALLOC_CTX *ctx, const char *options[])
+{
+
+	size_t num_options = 0;
+	const char **copy = NULL;
+	size_t i = 0;
+
+	if (options == NULL) {
+		return copy;
+	}
+
+	for (i=0; options[i]; i++) {
+		num_options++;
+	}
+
+	copy = talloc_zero_array(ctx, const char *, num_options + 1);
+	if (copy == NULL) {
+		return copy;
+	}
+
+	for (i=0; options[i]; i++) {
+		copy[i] = talloc_strdup(copy, options[i]);
+		if (copy[i] == NULL) {
+			TALLOC_FREE(copy);
+			return copy;
+		}
+	}
+	return copy;
+}
+
+const char **ldb_options_get(struct ldb_context *ldb)
+{
+	return ldb->options;
+}
diff --git a/lib/ldb/common/ldb_pack.c b/lib/ldb/common/ldb_pack.c
new file mode 100644
index 0000000..28b9a8d
--- /dev/null
+++ b/lib/ldb/common/ldb_pack.c
@@ -0,0 +1,1360 @@
+/*
+   ldb database library
+
+   Copyright (C) Andrew Tridgell  2004
+
+     ** NOTE! The following LGPL license applies to the ldb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+/*
+ *  Name: ldb
+ *
+ *  Component: ldb pack/unpack
+ *
+ *  Description: pack/unpack routines for ldb messages as key/value blobs
+ *
+ *  Author: Andrew Tridgell
+ */
+
+#include "ldb_private.h"
+
+/*
+ * These macros are from byte_array.h via libssh
+ * TODO: This will be replaced with use of the byte_array.h header when it
+ * becomes available.
+ *
+ * Macros for handling integer types in byte arrays
+ *
+ * This file is originally from the libssh.org project
+ *
+ * Copyright (c) 2018 Andreas Schneider <asn@cryptomilk.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#define _DATA_BYTE_CONST(data, pos) \
+	((uint8_t)(((const uint8_t *)(data))[(pos)]))
+#define PULL_LE_U8(data, pos) \
+	(_DATA_BYTE_CONST(data, pos))
+#define PULL_LE_U16(data, pos) \
+	((uint16_t)PULL_LE_U8(data, pos) |\
+	((uint16_t)(PULL_LE_U8(data, (pos) + 1))) << 8)
+#define PULL_LE_U32(data, pos) \
+	((uint32_t)(PULL_LE_U16(data, pos) |\
+	((uint32_t)PULL_LE_U16(data, (pos) + 2)) << 16))
+
+#define _DATA_BYTE(data, pos) \
+	(((uint8_t *)(data))[(pos)])
+#define PUSH_LE_U8(data, pos, val) \
+	(_DATA_BYTE(data, pos) = ((uint8_t)(val)))
+#define PUSH_LE_U16(data, pos, val) \
+	(PUSH_LE_U8((data), (pos), (uint8_t)((uint16_t)(val) & 0xff)),\
+		    PUSH_LE_U8((data), (pos) + 1,\
+			       (uint8_t)((uint16_t)(val) >> 8)))
+#define PUSH_LE_U32(data, pos, val) \
+	(PUSH_LE_U16((data), (pos), (uint16_t)((uint32_t)(val) & 0xffff)),\
+	 PUSH_LE_U16((data), (pos) + 2, (uint16_t)((uint32_t)(val) >> 16)))
+
+#define U32_LEN 4
+#define U16_LEN 2
+#define U8_LEN 1
+#define NULL_PAD_BYTE_LEN 1
+
+static int attribute_storable_values(const struct ldb_message_element *el)
+{
+	if (el->num_values == 0) return 0;
+
+	if (ldb_attr_cmp(el->name, "distinguishedName") == 0) return 0;
+
+	return el->num_values;
+}
+
+static int ldb_pack_data_v1(struct ldb_context *ldb,
+			    const struct ldb_message *message,
+			    struct ldb_val *data)
+{
+	unsigned int i, j, real_elements=0;
+	size_t size, dn_len, attr_len, value_len;
+	const char *dn;
+	uint8_t *p;
+	size_t len;
+
+	dn = ldb_dn_get_linearized(message->dn);
+	if (dn == NULL) {
+		errno = ENOMEM;
+		return -1;
+	}
+
+	/* work out how big it needs to be */
+	size = U32_LEN * 2 + NULL_PAD_BYTE_LEN;
+
+	dn_len = strlen(dn);
+	if (size + dn_len < size) {
+		errno = ENOMEM;
+		return -1;
+	}
+	size += dn_len;
+
+	/*
+	 * First calcuate the buffer size we need, and check for
+	 * overflows
+	 */
+	for (i=0;i<message->num_elements;i++) {
+		if (attribute_storable_values(&message->elements[i]) == 0) {
+			continue;
+		}
+
+		real_elements++;
+
+		if (size + U32_LEN + NULL_PAD_BYTE_LEN < size) {
+			errno = ENOMEM;
+			return -1;
+		}
+		size += U32_LEN + NULL_PAD_BYTE_LEN;
+
+		attr_len = strlen(message->elements[i].name);
+		if (size + attr_len < size) {
+			errno = ENOMEM;
+			return -1;
+		}
+		size += attr_len;
+
+		for (j=0;j<message->elements[i].num_values;j++) {
+			if (size + U32_LEN + NULL_PAD_BYTE_LEN < size) {
+				errno = ENOMEM;
+				return -1;
+			}
+			size += U32_LEN + NULL_PAD_BYTE_LEN;
+
+			value_len = message->elements[i].values[j].length;
+			if (size + value_len < size) {
+				errno = ENOMEM;
+				return -1;
+			}
+			size += value_len;
+		}
+	}
+
+	/* allocate it */
+	data->data = talloc_array(ldb, uint8_t, size);
+	if (!data->data) {
+		errno = ENOMEM;
+		return -1;
+	}
+	data->length = size;
+
+	p = data->data;
+	PUSH_LE_U32(p, 0, LDB_PACKING_FORMAT);
+	p += U32_LEN;
+	PUSH_LE_U32(p, 0, real_elements);
+	p += U32_LEN;
+
+	/* the dn needs to be packed so we can be case preserving
+	   while hashing on a case folded dn */
+	len = dn_len;
+	memcpy(p, dn, len+NULL_PAD_BYTE_LEN);
+	p += len + NULL_PAD_BYTE_LEN;
+
+	for (i=0;i<message->num_elements;i++) {
+		if (attribute_storable_values(&message->elements[i]) == 0) {
+			continue;
+		}
+		len = strlen(message->elements[i].name);
+		memcpy(p, message->elements[i].name, len+NULL_PAD_BYTE_LEN);
+		p += len + NULL_PAD_BYTE_LEN;
+		PUSH_LE_U32(p, 0, message->elements[i].num_values);
+		p += U32_LEN;
+		for (j=0;j<message->elements[i].num_values;j++) {
+			PUSH_LE_U32(p, 0,
+				    message->elements[i].values[j].length);
+			p += U32_LEN;
+			memcpy(p, message->elements[i].values[j].data,
+			       message->elements[i].values[j].length);
+			p[message->elements[i].values[j].length] = 0;
+			p += message->elements[i].values[j].length +
+				NULL_PAD_BYTE_LEN;
+		}
+	}
+
+	return 0;
+}
+
+/*
+ * New pack version designed based on performance profiling of version 1.
+ * The approach is to separate value data from the rest of the record's data.
+ * This improves performance because value data is not needed during unpacking
+ * or filtering of the message's attribute list. During filtering we only copy
+ * attributes which are present in the attribute list, however at the parse
+ * stage we need to point to all attributes as they may be referenced in the
+ * search expression.
+ * With this new format, we don't lose time loading data (eg via
+ * talloc_memdup()) that is never needed (for the vast majority of attributes
+ * are are never found in either the search expression or attribute list).
+ * Additional changes include adding a canonicalized DN (for later
+ * optimizations) and variable width length fields for faster unpacking.
+ * The pack and unpack performance improvement is tested in the torture
+ * test torture_ldb_pack_format_perf.
+ *
+ * Layout:
+ *
+ * Version (4 bytes)
+ * Number of Elements (4 bytes)
+ * DN length (4 bytes)
+ * DN with null terminator (DN length + 1 bytes)
+ * Canonicalized DN length (4 bytes)
+ * Canonicalized DN with null terminator (Canonicalized DN length + 1 bytes)
+ * Number of bytes from here to value data section (4 bytes)
+ * # For each element:
+ * 	Element name length (4 bytes)
+ * 	Element name with null terminator (Element name length + 1 bytes)
+ * 	Number of values (4 bytes)
+ * 	Width of value lengths
+ * 	# For each value:
+ * 		Value data length (#bytes given by width field above)
+ * # For each element:
+ * 	# For each value:
+ *	 	Value data (#bytes given by corresponding length above)
+ */
+static int ldb_pack_data_v2(struct ldb_context *ldb,
+			    const struct ldb_message *message,
+			    struct ldb_val *data)
+{
+	unsigned int i, j, real_elements=0;
+	size_t size, dn_len, dn_canon_len, attr_len, value_len;
+	const char *dn, *dn_canon;
+	uint8_t *p, *q;
+	size_t len;
+	size_t max_val_len;
+	uint8_t val_len_width;
+
+	/*
+	 * First half of this function will calculate required size for
+	 * packed data. Initial size is 20 = 5 * 4.  5 fixed fields are:
+	 * version, num elements, dn len, canon dn len, attr section len
+	 */
+	size = U32_LEN * 5;
+
+	/*
+	 * Get linearized and canonicalized form of the DN and add the lengths
+	 * of each to size, plus 1 for null terminator.
+	 */
+	dn = ldb_dn_get_linearized(message->dn);
+	if (dn == NULL) {
+		errno = ENOMEM;
+		return -1;
+	}
+
+	dn_len = strlen(dn) + NULL_PAD_BYTE_LEN;
+	if (size + dn_len < size) {
+		errno = ENOMEM;
+		return -1;
+	}
+	size += dn_len;
+
+	if (ldb_dn_is_special(message->dn)) {
+		dn_canon_len = NULL_PAD_BYTE_LEN;
+		dn_canon = discard_const_p(char, "\0");
+	} else {
+		dn_canon = ldb_dn_canonical_string(message->dn, message->dn);
+		if (dn_canon == NULL) {
+			errno = ENOMEM;
+			return -1;
+		}
+
+		dn_canon_len = strlen(dn_canon) + NULL_PAD_BYTE_LEN;
+		if (size + dn_canon_len < size) {
+			errno = ENOMEM;
+			return -1;
+		}
+	}
+	size += dn_canon_len;
+
+	/* Add the size required by each element */
+	for (i=0;i<message->num_elements;i++) {
+		if (attribute_storable_values(&message->elements[i]) == 0) {
+			continue;
+		}
+
+		real_elements++;
+
+		/*
+		 * Add length of element name + 9 for:
+		 * 1 for null terminator
+		 * 4 for element name length field
+		 * 4 for number of values field
+		 */
+		attr_len = strlen(message->elements[i].name);
+		if (size + attr_len + U32_LEN * 2 + NULL_PAD_BYTE_LEN < size) {
+			errno = ENOMEM;
+			return -1;
+		}
+		size += attr_len + U32_LEN * 2 + NULL_PAD_BYTE_LEN;
+
+		/*
+		 * Find the max value length, so we can calculate the width
+		 * required for the value length fields.
+		 */
+		max_val_len = 0;
+		for (j=0;j<message->elements[i].num_values;j++) {
+			value_len = message->elements[i].values[j].length;
+			if (value_len > max_val_len) {
+				max_val_len = value_len;
+			}
+
+			if (size + value_len + NULL_PAD_BYTE_LEN < size) {
+				errno = ENOMEM;
+				return -1;
+			}
+			size += value_len + NULL_PAD_BYTE_LEN;
+		}
+
+		if (max_val_len <= UCHAR_MAX) {
+			val_len_width = U8_LEN;
+		} else if (max_val_len <= USHRT_MAX) {
+			val_len_width = U16_LEN;
+		} else if (max_val_len <= UINT_MAX) {
+		        val_len_width = U32_LEN;
+		} else {
+			errno = EMSGSIZE;
+			return -1;
+		}
+
+		/* Total size required for val lengths (re-using variable) */
+		max_val_len = (val_len_width*message->elements[i].num_values);
+
+		/* Add one for storing the width */
+		max_val_len += U8_LEN;
+		if (size + max_val_len < size) {
+			errno = ENOMEM;
+			return -1;
+		}
+		size += max_val_len;
+	}
+
+	/* Allocate */
+	data->data = talloc_array(ldb, uint8_t, size);
+	if (!data->data) {
+		errno = ENOMEM;
+		return -1;
+	}
+	data->length = size;
+
+	/* Packing format version and number of element */
+	p = data->data;
+	PUSH_LE_U32(p, 0, LDB_PACKING_FORMAT_V2);
+	p += U32_LEN;
+	PUSH_LE_U32(p, 0, real_elements);
+	p += U32_LEN;
+
+	/* Pack DN and Canonicalized DN */
+	PUSH_LE_U32(p, 0, dn_len-NULL_PAD_BYTE_LEN);
+	p += U32_LEN;
+	memcpy(p, dn, dn_len);
+	p += dn_len;
+
+	PUSH_LE_U32(p, 0, dn_canon_len-NULL_PAD_BYTE_LEN);
+	p += U32_LEN;
+	memcpy(p, dn_canon, dn_canon_len);
+	p += dn_canon_len;
+
+	/*
+	 * Save pointer at this point and leave a U32_LEN gap for
+	 * storing the size of the attribute names and value lengths
+	 * section
+	 */
+	q = p;
+	p += U32_LEN;
+
+	for (i=0;i<message->num_elements;i++) {
+		if (attribute_storable_values(&message->elements[i]) == 0) {
+			continue;
+		}
+
+		/* Length of el name */
+		len = strlen(message->elements[i].name);
+		PUSH_LE_U32(p, 0, len);
+		p += U32_LEN;
+
+		/*
+		 * Even though we have the element name's length, put a null
+		 * terminator at the end so if any code uses the name
+		 * directly, it'll be safe to do things requiring null
+		 * termination like strlen
+		 */
+		memcpy(p, message->elements[i].name, len+NULL_PAD_BYTE_LEN);
+		p += len + NULL_PAD_BYTE_LEN;
+		/* Num values */
+		PUSH_LE_U32(p, 0, message->elements[i].num_values);
+		p += U32_LEN;
+
+		/*
+		 * Calculate value length width again. It's faster to
+		 * calculate it again than do the array management to
+		 * store the result during size calculation.
+		 */
+		max_val_len = 0;
+		for (j=0;j<message->elements[i].num_values;j++) {
+			value_len = message->elements[i].values[j].length;
+			if (value_len > max_val_len) {
+				max_val_len = value_len;
+			}
+		}
+
+		if (max_val_len <= UCHAR_MAX) {
+			val_len_width = U8_LEN;
+		} else if (max_val_len <= USHRT_MAX) {
+			val_len_width = U16_LEN;
+		} else if (max_val_len <= UINT_MAX) {
+		        val_len_width = U32_LEN;
+		} else {
+			errno = EMSGSIZE;
+			return -1;
+		}
+
+		/* Pack the width */
+		*p = val_len_width & 0xFF;
+		p += U8_LEN;
+
+		/*
+		 * Pack each value's length using the minimum number of bytes
+		 * required, which we just calculated. We repeat the loop
+		 * for each case here so the compiler can inline code.
+		 */
+		if (val_len_width == U8_LEN) {
+			for (j=0;j<message->elements[i].num_values;j++) {
+				PUSH_LE_U8(p, 0,
+					message->elements[i].values[j].length);
+				p += U8_LEN;
+			}
+		} else if (val_len_width == U16_LEN) {
+			for (j=0;j<message->elements[i].num_values;j++) {
+				PUSH_LE_U16(p, 0,
+					message->elements[i].values[j].length);
+				p += U16_LEN;
+			}
+		} else if (val_len_width == U32_LEN) {
+			for (j=0;j<message->elements[i].num_values;j++) {
+				PUSH_LE_U32(p, 0,
+					message->elements[i].values[j].length);
+				p += U32_LEN;
+			}
+		}
+	}
+
+	/*
+	 * We've finished packing the attr names and value lengths
+	 * section, so store the size in the U32_LEN gap we left
+	 * earlier
+	 */
+	PUSH_LE_U32(q, 0, p-q);
+
+	/* Now pack the values */
+	for (i=0;i<message->num_elements;i++) {
+		if (attribute_storable_values(&message->elements[i]) == 0) {
+			continue;
+		}
+		for (j=0;j<message->elements[i].num_values;j++) {
+			memcpy(p, message->elements[i].values[j].data,
+			       message->elements[i].values[j].length);
+
+			/*
+			 * Even though we have the data length, put a null
+			 * terminator at the end of each value's data so if
+			 * any code uses the data directly, it'll  be safe to
+			 * do things requiring null termination like strlen.
+			 */
+			p[message->elements[i].values[j].length] = 0;
+			p += message->elements[i].values[j].length +
+				NULL_PAD_BYTE_LEN;
+		}
+	}
+
+	/*
+	 * If we didn't end up at the end of the data here, something has
+	 * gone very wrong.
+	 */
+	if (p != data->data + size) {
+		errno = ENOMEM;
+		return -1;
+	}
+
+	return 0;
+}
+
+/*
+  pack a ldb message into a linear buffer in a ldb_val
+
+  note that this routine avoids saving elements with zero values,
+  as these are equivalent to having no element
+
+  caller frees the data buffer after use
+*/
+int ldb_pack_data(struct ldb_context *ldb,
+		  const struct ldb_message *message,
+		  struct ldb_val *data,
+		  uint32_t pack_format_version) {
+
+	if (pack_format_version == LDB_PACKING_FORMAT) {
+		return ldb_pack_data_v1(ldb, message, data);
+	} else if (pack_format_version == LDB_PACKING_FORMAT_V2) {
+		return ldb_pack_data_v2(ldb, message, data);
+	} else {
+		errno = EINVAL;
+		return -1;
+	}
+}
+
+/*
+ * Unpack a ldb message from a linear buffer in ldb_val
+ */
+static int ldb_unpack_data_flags_v1(struct ldb_context *ldb,
+				    const struct ldb_val *data,
+				    struct ldb_message *message,
+				    unsigned int flags,
+				    unsigned format)
+{
+	uint8_t *p;
+	size_t remaining;
+	size_t dn_len;
+	unsigned int i, j;
+	unsigned int nelem = 0;
+	size_t len;
+	struct ldb_val *ldb_val_single_array = NULL;
+
+	message->elements = NULL;
+
+	p = data->data;
+
+	/* Format (U32, already read) + U32 for num_elements */
+	if (data->length < U32_LEN * 2) {
+		errno = EIO;
+		goto failed;
+	}
+
+	/* Skip first 4 bytes, format already read */
+	p += U32_LEN;
+	message->num_elements = PULL_LE_U32(p, 0);
+	p += U32_LEN;
+
+	remaining = data->length - U32_LEN * 2;
+
+	switch (format) {
+	case LDB_PACKING_FORMAT_NODN:
+		message->dn = NULL;
+		break;
+
+	case LDB_PACKING_FORMAT:
+		/*
+		 * With this check, we know that the DN at p is \0
+		 * terminated.
+		 */
+		dn_len = strnlen((char *)p, remaining);
+		if (dn_len == remaining) {
+			errno = EIO;
+			goto failed;
+		}
+		if (flags & LDB_UNPACK_DATA_FLAG_NO_DN) {
+			message->dn = NULL;
+		} else {
+			struct ldb_val blob;
+			blob.data = discard_const_p(uint8_t, p);
+			blob.length = dn_len;
+			message->dn = ldb_dn_from_ldb_val(message, ldb, &blob);
+			if (message->dn == NULL) {
+				errno = ENOMEM;
+				goto failed;
+			}
+		}
+		/*
+		 * Redundant: by definition, remaining must be more
+		 * than one less than dn_len, as otherwise it would be
+		 * == dn_len
+		 */
+		if (remaining < dn_len + NULL_PAD_BYTE_LEN) {
+			errno = EIO;
+			goto failed;
+		}
+		remaining -= dn_len + NULL_PAD_BYTE_LEN;
+		p += dn_len + NULL_PAD_BYTE_LEN;
+		break;
+
+	default:
+		errno = EIO;
+		goto failed;
+	}
+
+	if (flags & LDB_UNPACK_DATA_FLAG_NO_ATTRS) {
+		return 0;
+	}
+	
+	if (message->num_elements == 0) {
+		return 0;
+	}
+
+	if (message->num_elements > remaining / 6) {
+		errno = EIO;
+		goto failed;
+	}
+
+	message->elements = talloc_zero_array(message, struct ldb_message_element,
+					      message->num_elements);
+	if (!message->elements) {
+		errno = ENOMEM;
+		goto failed;
+	}
+
+	/*
+	 * In typical use, most values are single-valued.  This makes
+	 * it quite expensive to allocate an array of ldb_val for each
+	 * of these, just to then hold the pointer to the data buffer
+	 * So with LDB_UNPACK_DATA_FLAG_NO_VALUES_ALLOC we allocate this
+	 * ahead of time and use it for the single values where possible.
+	 * (This is used the the normal search case, but not in the
+	 * index case because of caller requirements).
+	 */
+	if (flags & LDB_UNPACK_DATA_FLAG_NO_VALUES_ALLOC) {
+		ldb_val_single_array = talloc_array(message->elements, struct ldb_val,
+						    message->num_elements);
+		if (ldb_val_single_array == NULL) {
+			errno = ENOMEM;
+			goto failed;
+		}
+	}
+
+	for (i=0;i<message->num_elements;i++) {
+		const char *attr = NULL;
+		size_t attr_len;
+		struct ldb_message_element *element = NULL;
+
+		/*
+		 * Sanity check: Element must be at least the size of empty
+		 * attr name and value and NULL terms for each.
+		 */
+		if (remaining < U32_LEN * 2 + NULL_PAD_BYTE_LEN * 2) {
+			errno = EIO;
+			goto failed;
+		}
+
+		/*
+		 * With this check, we know that the attribute name at
+		 * p is \0 terminated.
+		 */
+		attr_len = strnlen((char *)p, remaining-6);
+		if (attr_len == remaining-6) {
+			errno = EIO;
+			goto failed;
+		}
+		if (attr_len == 0) {
+			errno = EIO;
+			goto failed;
+		}
+		attr = (char *)p;
+
+		element = &message->elements[nelem];
+		element->name = attr;
+		element->flags = 0;
+
+		if (remaining < (attr_len + NULL_PAD_BYTE_LEN)) {
+			errno = EIO;
+			goto failed;
+		}
+		remaining -= attr_len + NULL_PAD_BYTE_LEN;
+		p += attr_len + NULL_PAD_BYTE_LEN;
+		element->num_values = PULL_LE_U32(p, 0);
+		element->values = NULL;
+		if ((flags & LDB_UNPACK_DATA_FLAG_NO_VALUES_ALLOC) && element->num_values == 1) {
+			element->values = &ldb_val_single_array[nelem];
+			element->flags |= LDB_FLAG_INTERNAL_SHARED_VALUES;
+		} else if (element->num_values != 0) {
+			element->values = talloc_array(message->elements,
+						       struct ldb_val,
+						       element->num_values);
+			if (!element->values) {
+				errno = ENOMEM;
+				goto failed;
+			}
+		}
+		p += U32_LEN;
+		if (remaining < U32_LEN) {
+			errno = EIO;
+			goto failed;
+		}
+		remaining -= U32_LEN;
+		for (j = 0; j < element->num_values; j++) {
+			/*
+			 * Sanity check: Value must be at least the size of
+			 * empty val and NULL terminator.
+			 */
+			if (remaining < U32_LEN + NULL_PAD_BYTE_LEN) {
+				errno = EIO;
+				goto failed;
+			}
+			remaining -= U32_LEN + NULL_PAD_BYTE_LEN;
+
+			len = PULL_LE_U32(p, 0);
+			if (remaining < len) {
+				errno = EIO;
+				goto failed;
+			}
+			if (len + NULL_PAD_BYTE_LEN < len) {
+				errno = EIO;
+				goto failed;
+			}
+
+			element->values[j].length = len;
+			element->values[j].data = p + U32_LEN;
+			remaining -= len;
+			p += len + U32_LEN + NULL_PAD_BYTE_LEN;
+		}
+		nelem++;
+	}
+	/*
+	 * Adapt the number of elements to the real number of unpacked elements,
+	 * it means that we overallocated elements array.
+	 */
+	message->num_elements = nelem;
+
+	/*
+	 * Shrink the allocated size.  On current talloc behaviour
+	 * this will help if we skipped 32 or more attributes.
+	 */
+	message->elements = talloc_realloc(message, message->elements,
+					   struct ldb_message_element,
+					   message->num_elements);
+
+	if (remaining != 0) {
+		ldb_debug(ldb, LDB_DEBUG_ERROR,
+			  "Error: %zu bytes unread in ldb_unpack_data_flags",
+			  remaining);
+	}
+
+	return 0;
+
+failed:
+	talloc_free(message->elements);
+	return -1;
+}
+
+/*
+ * Unpack a ldb message from a linear buffer in ldb_val
+ */
+static int ldb_unpack_data_flags_v2(struct ldb_context *ldb,
+				    const struct ldb_val *data,
+				    struct ldb_message *message,
+				    unsigned int flags)
+{
+	uint8_t *p, *q, *end_p, *value_section_p;
+	unsigned int i, j;
+	unsigned int nelem = 0;
+	size_t len;
+	struct ldb_val *ldb_val_single_array = NULL;
+	uint8_t val_len_width;
+
+	message->elements = NULL;
+
+	p = data->data;
+	end_p = p + data->length;
+
+	/* Skip first 4 bytes, format already read */
+	p += U32_LEN;
+
+	/* First fields are fixed: num_elements, DN length */
+	if (p + U32_LEN * 2 > end_p) {
+		errno = EIO;
+		goto failed;
+	}
+
+	message->num_elements = PULL_LE_U32(p, 0);
+	p += U32_LEN;
+
+	len = PULL_LE_U32(p, 0);
+	p += U32_LEN;
+
+	if (p + len + NULL_PAD_BYTE_LEN > end_p) {
+		errno = EIO;
+		goto failed;
+	}
+
+	if (flags & LDB_UNPACK_DATA_FLAG_NO_DN) {
+		message->dn = NULL;
+	} else {
+		struct ldb_val blob;
+		blob.data = discard_const_p(uint8_t, p);
+		blob.length = len;
+		message->dn = ldb_dn_from_ldb_val(message, ldb, &blob);
+		if (message->dn == NULL) {
+			errno = ENOMEM;
+			goto failed;
+		}
+	}
+
+	p += len + NULL_PAD_BYTE_LEN;
+
+	if (*(p-NULL_PAD_BYTE_LEN) != '\0') {
+		errno = EINVAL;
+		goto failed;
+	}
+
+	/* Now skip the canonicalized DN and its length */
+	len = PULL_LE_U32(p, 0) + NULL_PAD_BYTE_LEN;
+	p += U32_LEN;
+
+	if (p + len > end_p) {
+		errno = EIO;
+		goto failed;
+	}
+
+	p += len;
+
+	if (*(p-NULL_PAD_BYTE_LEN) != '\0') {
+		errno = EINVAL;
+		goto failed;
+	}
+
+	if (flags & LDB_UNPACK_DATA_FLAG_NO_ATTRS) {
+		return 0;
+	}
+
+	if (message->num_elements == 0) {
+		return 0;
+	}
+
+	/*
+	 * Sanity check (17 bytes is the minimum element size)
+	 */
+	if (message->num_elements > (end_p - p) / 17) {
+		errno = EIO;
+		goto failed;
+	}
+
+	message->elements = talloc_zero_array(message,
+					      struct ldb_message_element,
+					      message->num_elements);
+	if (!message->elements) {
+		errno = ENOMEM;
+		goto failed;
+	}
+
+	/*
+	 * In typical use, most values are single-valued.  This makes
+	 * it quite expensive to allocate an array of ldb_val for each
+	 * of these, just to then hold the pointer to the data buffer.
+	 * So with LDB_UNPACK_DATA_FLAG_NO_VALUES_ALLOC we allocate this
+	 * ahead of time and use it for the single values where possible.
+	 * (This is used the the normal search case, but not in the
+	 * index case because of caller requirements).
+	 */
+	if (flags & LDB_UNPACK_DATA_FLAG_NO_VALUES_ALLOC) {
+		ldb_val_single_array = talloc_array(message->elements,
+						    struct ldb_val,
+						    message->num_elements);
+		if (ldb_val_single_array == NULL) {
+			errno = ENOMEM;
+			goto failed;
+		}
+	}
+
+	q = p + PULL_LE_U32(p, 0);
+	value_section_p = q;
+	p += U32_LEN;
+
+	for (i=0;i<message->num_elements;i++) {
+		const char *attr = NULL;
+		size_t attr_len;
+		struct ldb_message_element *element = NULL;
+
+		/* Sanity check: minimum element size */
+		if (p + (U32_LEN * 2) + /* attr name len, num values */
+			(U8_LEN * 2) + /* value length width, one val length */
+			(NULL_PAD_BYTE_LEN * 2) /* null for attr name + val */
+			> value_section_p) {
+			errno = EIO;
+			goto failed;
+		}
+
+		attr_len = PULL_LE_U32(p, 0);
+		p += U32_LEN;
+
+		if (attr_len == 0) {
+			errno = EIO;
+			goto failed;
+		}
+		attr = (char *)p;
+
+		p += attr_len + NULL_PAD_BYTE_LEN;
+		/*
+		 * num_values, val_len_width
+		 *
+		 * val_len_width is the width specifier
+		 * for the variable length encoding
+		 */
+		if (p + U32_LEN + U8_LEN > value_section_p) {
+			errno = EIO;
+			goto failed;
+		}
+
+		if (*(p-NULL_PAD_BYTE_LEN) != '\0') {
+			errno = EINVAL;
+			goto failed;
+		}
+
+		element = &message->elements[nelem];
+		element->name = attr;
+		element->flags = 0;
+
+		element->num_values = PULL_LE_U32(p, 0);
+		element->values = NULL;
+		if ((flags & LDB_UNPACK_DATA_FLAG_NO_VALUES_ALLOC) &&
+		    element->num_values == 1) {
+			element->values = &ldb_val_single_array[nelem];
+			element->flags |= LDB_FLAG_INTERNAL_SHARED_VALUES;
+		} else if (element->num_values != 0) {
+			element->values = talloc_array(message->elements,
+						       struct ldb_val,
+						       element->num_values);
+			if (!element->values) {
+				errno = ENOMEM;
+				goto failed;
+			}
+		}
+
+		p += U32_LEN;
+
+		/*
+		 * Here we read how wide the remaining lengths are
+		 * which avoids storing and parsing a lot of leading
+		 * 0s
+		 */
+		val_len_width = *p;
+		p += U8_LEN;
+
+		if (p + val_len_width * element->num_values >
+		    value_section_p) {
+			errno = EIO;
+			goto failed;
+		}
+
+		/*
+		 * This is structured weird for compiler optimization
+		 * purposes, but we need to pull the array of widths
+		 * with different macros depending on how wide the
+		 * biggest one is (specified by val_len_width)
+		 */
+		if (val_len_width == U8_LEN) {
+			for (j = 0; j < element->num_values; j++) {
+				element->values[j].length = PULL_LE_U8(p, 0);
+				p += U8_LEN;
+			}
+		} else if (val_len_width == U16_LEN) {
+			for (j = 0; j < element->num_values; j++) {
+				element->values[j].length = PULL_LE_U16(p, 0);
+				p += U16_LEN;
+			}
+		} else if (val_len_width == U32_LEN) {
+			for (j = 0; j < element->num_values; j++) {
+				element->values[j].length = PULL_LE_U32(p, 0);
+				p += U32_LEN;
+			}
+		} else {
+			errno = ERANGE;
+			goto failed;
+		}
+
+		for (j = 0; j < element->num_values; j++) {
+			len = element->values[j].length;
+			if (len + NULL_PAD_BYTE_LEN < len) {
+				errno = EIO;
+				goto failed;
+			}
+			if (q + len + NULL_PAD_BYTE_LEN > end_p) {
+				errno = EIO;
+				goto failed;
+			}
+
+			element->values[j].data = q;
+			q += len + NULL_PAD_BYTE_LEN;
+		}
+		nelem++;
+	}
+
+	/*
+	 * If p isn't now pointing at the beginning of the value section,
+	 * something went very wrong.
+	 */
+	if (p != value_section_p) {
+		ldb_debug(ldb, LDB_DEBUG_ERROR,
+			  "Error: Data corruption in ldb_unpack_data_flags");
+		errno = EIO;
+		goto failed;
+	}
+
+	/*
+	 * Adapt the number of elements to the real number of unpacked
+	 * elements it means that we overallocated elements array.
+	 */
+	message->num_elements = nelem;
+
+	/*
+	 * Shrink the allocated size.  On current talloc behaviour
+	 * this will help if we skipped 32 or more attributes.
+	 */
+	message->elements = talloc_realloc(message, message->elements,
+					   struct ldb_message_element,
+					   message->num_elements);
+
+	if (q != end_p) {
+		ldb_debug(ldb, LDB_DEBUG_ERROR,
+			  "Error: %zu bytes unread in ldb_unpack_data_flags",
+			  end_p - q);
+		errno = EIO;
+		goto failed;
+	}
+
+	return 0;
+
+failed:
+	talloc_free(message->elements);
+	return -1;
+}
+
+int ldb_unpack_get_format(const struct ldb_val *data,
+			  uint32_t *pack_format_version)
+{
+	if (data->length < U32_LEN) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+	*pack_format_version = PULL_LE_U32(data->data, 0);
+	return LDB_SUCCESS;
+}
+
+/*
+ * Unpack a ldb message from a linear buffer in ldb_val
+ */
+int ldb_unpack_data_flags(struct ldb_context *ldb,
+			  const struct ldb_val *data,
+			  struct ldb_message *message,
+			  unsigned int flags)
+{
+	unsigned format;
+
+	if (data->length < U32_LEN) {
+		errno = EIO;
+		return -1;
+	}
+
+	format = PULL_LE_U32(data->data, 0);
+	if (format == LDB_PACKING_FORMAT_V2) {
+		return ldb_unpack_data_flags_v2(ldb, data, message, flags);
+	}
+
+	/*
+	 * The v1 function we're about to call takes either LDB_PACKING_FORMAT
+	 * or LDB_PACKING_FORMAT_NODN packing format versions, and will error
+	 * if given some other version, so we don't need to do any further
+	 * checks on 'format'.
+	 */
+	return ldb_unpack_data_flags_v1(ldb, data, message, flags, format);
+}
+
+
+/*
+ * Unpack a ldb message from a linear buffer in ldb_val
+ *
+ * Free with ldb_unpack_data_free()
+ */
+int ldb_unpack_data(struct ldb_context *ldb,
+		    const struct ldb_val *data,
+		    struct ldb_message *message)
+{
+	return ldb_unpack_data_flags(ldb, data, message, 0);
+}
+
+/*
+  add the special distinguishedName element
+*/
+int ldb_msg_add_distinguished_name(struct ldb_message *msg)
+{
+	const char *dn_attr = "distinguishedName";
+	char *dn = NULL;
+
+	if (ldb_msg_find_element(msg, dn_attr)) {
+		/*
+		 * This should not happen, but this is
+		 * existing behaviour...
+		 */
+		return LDB_SUCCESS;
+	}
+
+	dn = ldb_dn_alloc_linearized(msg, msg->dn);
+	if (dn == NULL) {
+		return LDB_ERR_OPERATIONS_ERROR;
+	}
+
+	return ldb_msg_add_steal_string(msg, dn_attr, dn);
+}
+
+/*
+ * filter the specified list of attributes from msg,
+ * adding requested attributes, and perhaps all for *,
+ * but not the DN to filtered_msg.
+ */
+int ldb_filter_attrs(struct ldb_context *ldb,
+		     const struct ldb_message *msg,
+		     const char *const *attrs,
+		     struct ldb_message *filtered_msg)
+{
+	unsigned int i;
+	bool keep_all = false;
+	bool add_dn = false;
+	uint32_t num_elements;
+	uint32_t elements_size;
+
+	if (attrs) {
+		/* check for special attrs */
+		for (i = 0; attrs[i]; i++) {
+			int cmp = strcmp(attrs[i], "*");
+			if (cmp == 0) {
+				keep_all = true;
+				break;
+			}
+			cmp = ldb_attr_cmp(attrs[i], "distinguishedName");
+			if (cmp == 0) {
+				add_dn = true;
+			}
+		}
+	} else {
+		keep_all = true;
+	}
+
+	if (keep_all) {
+		add_dn = true;
+		elements_size = msg->num_elements + 1;
+
+	/* Shortcuts for the simple cases */
+	} else if (add_dn && i == 1) {
+		if (ldb_msg_add_distinguished_name(filtered_msg) != 0) {
+			goto failed;
+		}
+		return 0;
+	} else if (i == 0) {
+		return 0;
+
+	/*
+	 * Otherwise we are copying at most as many elements as we
+	 * have attributes
+	 */
+	} else {
+		elements_size = i;
+	}
+
+	filtered_msg->elements = talloc_array(filtered_msg,
+					      struct ldb_message_element,
+					      elements_size);
+	if (filtered_msg->elements == NULL) goto failed;
+
+	num_elements = 0;
+
+	for (i = 0; i < msg->num_elements; i++) {
+		struct ldb_message_element *el = &msg->elements[i];
+
+		/*
+		 * el2 is assigned after the Pigeonhole principle
+		 * check below for clarity
+		 */
+		struct ldb_message_element *el2 = NULL;
+		unsigned int j;
+
+		if (keep_all == false) {
+			bool found = false;
+			for (j = 0; attrs[j]; j++) {
+				int cmp = ldb_attr_cmp(el->name, attrs[j]);
+				if (cmp == 0) {
+					found = true;
+					break;
+				}
+			}
+			if (found == false) {
+				continue;
+			}
+		}
+
+		/*
+		 * Pigeonhole principle: we can't have more elements
+		 * than the number of attributes if they are unique in
+		 * the DB.
+		 */
+		if (num_elements >= elements_size) {
+			goto failed;
+		}
+
+		el2 = &filtered_msg->elements[num_elements];
+
+		*el2 = *el;
+		el2->name = talloc_strdup(filtered_msg->elements,
+					  el->name);
+		if (el2->name == NULL) {
+			goto failed;
+		}
+		el2->values = talloc_array(filtered_msg->elements,
+					   struct ldb_val, el->num_values);
+		if (el2->values == NULL) {
+			goto failed;
+		}
+		for (j=0;j<el->num_values;j++) {
+			el2->values[j] = ldb_val_dup(el2->values, &el->values[j]);
+			if (el2->values[j].data == NULL && el->values[j].length != 0) {
+				goto failed;
+			}
+		}
+		num_elements++;
+	}
+
+	filtered_msg->num_elements = num_elements;
+
+	if (add_dn) {
+		if (ldb_msg_add_distinguished_name(filtered_msg) != 0) {
+			goto failed;
+		}
+	}
+
+	if (filtered_msg->num_elements > 0) {
+		filtered_msg->elements
+			= talloc_realloc(filtered_msg,
+					 filtered_msg->elements,
+					 struct ldb_message_element,
+					 filtered_msg->num_elements);
+		if (filtered_msg->elements == NULL) {
+			goto failed;
+		}
+	} else {
+		TALLOC_FREE(filtered_msg->elements);
+	}
+
+	return 0;
+failed:
+	TALLOC_FREE(filtered_msg->elements);
+	return -1;
+}
+
+/*
+ * filter the specified list of attributes from msg,
+ * adding requested attributes, and perhaps all for *.
+ * Unlike ldb_filter_attrs(), the DN will not be added
+ * if it is missing.
+ */
+int ldb_filter_attrs_in_place(struct ldb_message *msg,
+			      const char *const *attrs)
+{
+	unsigned int i = 0;
+	bool keep_all = false;
+	unsigned int num_del = 0;
+
+	if (attrs) {
+		/* check for special attrs */
+		for (i = 0; attrs[i]; i++) {
+			int cmp = strcmp(attrs[i], "*");
+			if (cmp == 0) {
+				keep_all = true;
+				break;
+			}
+		}
+		if (!keep_all && i == 0) {
+			msg->num_elements = 0;
+			return LDB_SUCCESS;
+		}
+	} else {
+		keep_all = true;
+	}
+
+	for (i = 0; i < msg->num_elements; i++) {
+		bool found = false;
+		unsigned int j;
+
+		if (keep_all) {
+			found = true;
+		} else {
+			for (j = 0; attrs[j]; j++) {
+				int cmp = ldb_attr_cmp(msg->elements[i].name, attrs[j]);
+				if (cmp == 0) {
+					found = true;
+					break;
+				}
+			}
+		}
+
+		if (!found) {
+			++num_del;
+		} else if (num_del != 0) {
+			msg->elements[i - num_del] = msg->elements[i];
+		}
+	}
+
+	msg->num_elements -= num_del;
+
+	return LDB_SUCCESS;
+}
+
+/* Have an unpacked ldb message take talloc ownership of its elements. */
+int ldb_msg_elements_take_ownership(struct ldb_message *msg)
+{
+	unsigned int i = 0;
+
+	for (i = 0; i < msg->num_elements; i++) {
+		struct ldb_message_element *el = &msg->elements[i];
+		const char *name;
+		unsigned int j;
+
+		name = talloc_strdup(msg->elements,
+				     el->name);
+		if (name == NULL) {
+			return -1;
+		}
+		el->name = name;
+
+		if (el->flags & LDB_FLAG_INTERNAL_SHARED_VALUES) {
+			struct ldb_val *values = talloc_memdup(msg->elements, el->values,
+							       sizeof(struct ldb_val) * el->num_values);
+			if (values == NULL) {
+				return -1;
+			}
+			el->values = values;
+			el->flags &= ~LDB_FLAG_INTERNAL_SHARED_VALUES;
+		}
+
+		for (j = 0; j < el->num_values; j++) {
+			struct ldb_val val = ldb_val_dup(el->values, &el->values[j]);
+			if (val.data == NULL && el->values[j].length != 0) {
+				return -1;
+			}
+			el->values[j] = val;
+		}
+	}
+
+	return LDB_SUCCESS;
+}
diff --git a/lib/ldb/common/ldb_parse.c b/lib/ldb/common/ldb_parse.c
new file mode 100644
index 0000000..2d102ff
--- /dev/null
+++ b/lib/ldb/common/ldb_parse.c
@@ -0,0 +1,1024 @@
+/* 
+   ldb database library
+
+   Copyright (C) Andrew Tridgell  2004
+
+     ** NOTE! The following LGPL license applies to the ldb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+   
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+/*
+ *  Name: ldb
+ *
+ *  Component: ldb expression parsing
+ *
+ *  Description: parse LDAP-like search expressions
+ *
+ *  Author: Andrew Tridgell
+ */
+
+/*
+  TODO:
+      - add RFC2254 binary string handling
+      - possibly add ~=, <= and >= handling
+      - expand the test suite
+      - add better parse error handling
+
+*/
+
+#include "ldb_private.h"
+#include "system/locale.h"
+
+/*
+ * Maximum depth of the filter parse tree, the value chosen is small enough to
+ * avoid triggering ASAN stack overflow checks. But large enough to be useful.
+ *
+ * On Windows clients the maximum number of levels of recursion allowed is 100.
+ * In the LDAP server, Windows restricts clients to 512 nested
+ * (eg) OR statements.
+ */
+#define LDB_MAX_PARSE_TREE_DEPTH 128
+
+/*
+a filter is defined by:
+               <filter> ::= '(' <filtercomp> ')'
+               <filtercomp> ::= <and> | <or> | <not> | <simple>
+               <and> ::= '&' <filterlist>
+               <or> ::= '|' <filterlist>
+               <not> ::= '!' <filter>
+               <filterlist> ::= <filter> | <filter> <filterlist>
+               <simple> ::= <attributetype> <filtertype> <attributevalue>
+               <filtertype> ::= '=' | '~=' | '<=' | '>='
+*/
+
+/*
+   decode a RFC2254 binary string representation of a buffer.
+   Used in LDAP filters.
+*/
+struct ldb_val ldb_binary_decode(TALLOC_CTX *mem_ctx, const char *str)
+{
+	size_t i, j;
+	struct ldb_val ret;
+	size_t slen = str?strlen(str):0;
+
+	ret.data = (uint8_t *)talloc_size(mem_ctx, slen+1);
+	ret.length = 0;
+	if (ret.data == NULL) return ret;
+
+	for (i=j=0;i<slen;i++) {
+		if (str[i] == '\\') {
+			uint8_t c;
+			bool ok;
+
+			ok = hex_byte(&str[i+1], &c);
+			if (!ok) {
+				talloc_free(ret.data);
+				memset(&ret, 0, sizeof(ret));
+				return ret;
+			}
+			((uint8_t *)ret.data)[j++] = c;
+			i += 2;
+		} else {
+			((uint8_t *)ret.data)[j++] = str[i];
+		}
+	}
+	ret.length = j;
+	((uint8_t *)ret.data)[j] = 0;
+
+	return ret;
+}
+
+static bool need_encode(unsigned char cval)
+{
+	if (cval < 0x20 || cval > 0x7E || strchr(" *()\\&|!\"", cval)) {
+		return true;
+	}
+	return false;
+}
+
+/*
+   encode a blob as a RFC2254 binary string, escaping any
+   non-printable or '\' characters
+*/
+char *ldb_binary_encode(TALLOC_CTX *mem_ctx, struct ldb_val val)
+{
+	size_t i;
+	char *ret;
+	size_t len = val.length;
+	unsigned char *buf = val.data;
+
+	for (i=0;i<val.length;i++) {
+		if (need_encode(buf[i])) {
+			len += 2;
+		}
+	}
+	ret = talloc_array(mem_ctx, char, len+1);
+	if (ret == NULL) return NULL;
+
+	len = 0;
+	for (i=0;i<val.length;i++) {
+		if (need_encode(buf[i])) {
+			snprintf(ret+len, 4, "\\%02X", buf[i]);
+			len += 3;
+		} else {
+			ret[len++] = buf[i];
+		}
+	}
+
+	ret[len] = 0;
+
+	return ret;	
+}
+
+/*
+   encode a string as a RFC2254 binary string, escaping any
+   non-printable or '\' characters.  This routine is suitable for use
+   in escaping user data in ldap filters.
+*/
+char *ldb_binary_encode_string(TALLOC_CTX *mem_ctx, const char *string)
+{
+	struct ldb_val val;
+	if (string == NULL) {
+		return NULL;
+	}
+	val.data = discard_const_p(uint8_t, string);
+	val.length = strlen(string);
+	return ldb_binary_encode(mem_ctx, val);
+}
+
+/* find the first matching wildcard */
+static char *ldb_parse_find_wildcard(char *value)
+{
+	while (*value) {
+		value = strpbrk(value, "\\*");
+		if (value == NULL) return NULL;
+
+		if (value[0] == '\\') {
+			if (value[1] == '\0') return NULL;
+			value += 2;
+			continue;
+		}
+
+		if (value[0] == '*') return value;
+	}
+
+	return NULL;
+}
+
+/* return a NULL terminated list of binary strings representing the value
+   chunks separated by wildcards that makes the value portion of the filter
+*/
+static struct ldb_val **ldb_wildcard_decode(TALLOC_CTX *mem_ctx, const char *string)
+{
+	struct ldb_val **ret = NULL;
+	unsigned int val = 0;
+	char *wc, *str;
+
+	wc = talloc_strdup(mem_ctx, string);
+	if (wc == NULL) return NULL;
+
+	while (wc && *wc) {
+		str = wc;
+		wc = ldb_parse_find_wildcard(str);
+		if (wc && *wc) {
+			if (wc == str) {
+				wc++;
+				continue;
+			}
+			*wc = 0;
+			wc++;
+		}
+
+		ret = talloc_realloc(mem_ctx, ret, struct ldb_val *, val + 2);
+		if (ret == NULL) return NULL;
+
+		ret[val] = talloc(mem_ctx, struct ldb_val);
+		if (ret[val] == NULL) return NULL;
+
+		*(ret[val]) = ldb_binary_decode(mem_ctx, str);
+		if ((ret[val])->data == NULL) return NULL;
+
+		val++;
+	}
+
+	if (ret != NULL) {
+		ret[val] = NULL;
+	}
+
+	return ret;
+}
+
+static struct ldb_parse_tree *ldb_parse_filter(
+	TALLOC_CTX *mem_ctx,
+	const char **s,
+	unsigned depth,
+	unsigned max_depth);
+
+
+/*
+  parse an extended match
+
+  possible forms:
+        (attr:oid:=value)
+        (attr:dn:oid:=value)
+        (attr:dn:=value)
+        (:dn:oid:=value)
+
+  the ':dn' part sets the dnAttributes boolean if present
+  the oid sets the rule_id string
+  
+*/
+static struct ldb_parse_tree *ldb_parse_extended(struct ldb_parse_tree *ret, 
+						 char *attr, char *value)
+{
+	char *p1, *p2;
+
+	ret->operation = LDB_OP_EXTENDED;
+	ret->u.extended.value = ldb_binary_decode(ret, value);
+	if (ret->u.extended.value.data == NULL) goto failed;
+
+	p1 = strchr(attr, ':');
+	if (p1 == NULL) goto failed;
+	p2 = strchr(p1+1, ':');
+
+	*p1 = 0;
+	if (p2) *p2 = 0;
+
+	ret->u.extended.attr = attr;
+	if (strcmp(p1+1, "dn") == 0) {
+		ret->u.extended.dnAttributes = 1;
+		if (p2) {
+			ret->u.extended.rule_id = talloc_strdup(ret, p2+1);
+			if (ret->u.extended.rule_id == NULL) goto failed;
+		} else {
+			ret->u.extended.rule_id = NULL;
+		}
+	} else {
+		ret->u.extended.dnAttributes = 0;
+		ret->u.extended.rule_id = talloc_strdup(ret, p1+1);
+		if (ret->u.extended.rule_id == NULL) goto failed;
+	}
+
+	return ret;
+
+failed:
+	talloc_free(ret);
+	return NULL;
+}
+
+static enum ldb_parse_op ldb_parse_filtertype(TALLOC_CTX *mem_ctx, char **type, char **value, const char **s)
+{
+	enum ldb_parse_op filter = 0;
+	char *name, *val, *k;
+	const char *p = *s;
+	const char *t, *t1;
+
+	/* retrieve attributetype name */
+	t = p;
+
+	if (*p == '@') { /* for internal attributes the first char can be @ */
+		p++;
+	}
+
+	while ((isascii(*p) && isalnum((unsigned char)*p)) || (*p == '-') || (*p == '.')) { 
+		/* attribute names can only be alphanums */
+		p++;
+	}
+
+	if (*p == ':') { /* but extended searches have : and . chars too */
+		p = strstr(p, ":=");
+		if (p == NULL) { /* malformed attribute name */
+			return 0;
+		}
+	}
+
+	t1 = p;
+
+	while (isspace((unsigned char)*p)) p++;
+
+	if (!strchr("=<>~:", *p)) {
+		return 0;
+	}
+
+	/* save name */
+	name = (char *)talloc_memdup(mem_ctx, t, t1 - t + 1);
+	if (name == NULL) return 0;
+	name[t1 - t] = '\0';
+
+	/* retrieve filtertype */
+
+	if (*p == '=') {
+		filter = LDB_OP_EQUALITY;
+	} else if (*p != '\0' && *(p + 1) == '=') {
+		switch (*p) {
+		case '<':
+			filter = LDB_OP_LESS;
+			p++;
+			break;
+		case '>':
+			filter = LDB_OP_GREATER;
+			p++;
+			break;
+		case '~':
+			filter = LDB_OP_APPROX;
+			p++;
+			break;
+		case ':':
+			filter = LDB_OP_EXTENDED;
+			p++;
+			break;
+		}
+	}
+	if (!filter) {
+		talloc_free(name);
+		return 0;
+	}
+	p++;
+
+	while (isspace((unsigned char)*p)) p++;
+
+	/* retrieve value */
+	t = p;
+
+	while (*p && ((*p != ')') || ((*p == ')') && (*(p - 1) == '\\')))) p++;
+
+	val = (char *)talloc_memdup(mem_ctx, t, p - t + 1);
+	if (val == NULL) {
+		talloc_free(name);
+		return 0;
+	}
+	val[p - t] = '\0';
+
+	k = &(val[p - t]);
+
+	/* remove trailing spaces from value */
+	while ((k > val) && (isspace((unsigned char)*(k - 1)))) k--;
+	*k = '\0';
+
+	*type = name;
+	*value = val;
+	*s = p;
+	return filter;
+}
+
+/*
+  <simple> ::= <attributetype> <filtertype> <attributevalue>
+*/
+static struct ldb_parse_tree *ldb_parse_simple(TALLOC_CTX *mem_ctx, const char **s)
+{
+	char *attr, *value;
+	struct ldb_parse_tree *ret;
+	enum ldb_parse_op filtertype;
+
+	ret = talloc_zero(mem_ctx, struct ldb_parse_tree);
+	if (!ret) {
+		errno = ENOMEM;
+		return NULL;
+	}
+
+	filtertype = ldb_parse_filtertype(ret, &attr, &value, s);
+	if (!filtertype) {
+		talloc_free(ret);
+		return NULL;
+	}
+
+	switch (filtertype) {
+
+		case LDB_OP_PRESENT:
+			ret->operation = LDB_OP_PRESENT;
+			ret->u.present.attr = attr;
+			break;
+
+		case LDB_OP_EQUALITY:
+
+			if (strcmp(value, "*") == 0) {
+				ret->operation = LDB_OP_PRESENT;
+				ret->u.present.attr = attr;
+				break;
+			}
+
+			if (ldb_parse_find_wildcard(value) != NULL) {
+				ret->operation = LDB_OP_SUBSTRING;
+				ret->u.substring.attr = attr;
+				ret->u.substring.start_with_wildcard = 0;
+				ret->u.substring.end_with_wildcard = 0;
+				ret->u.substring.chunks = ldb_wildcard_decode(ret, value);
+				if (ret->u.substring.chunks == NULL){
+					talloc_free(ret);
+					return NULL;
+				}
+				if (value[0] == '*')
+					ret->u.substring.start_with_wildcard = 1;
+				if (value[strlen(value) - 1] == '*')
+					ret->u.substring.end_with_wildcard = 1;
+				talloc_free(value);
+
+				break;
+			}
+
+			ret->operation = LDB_OP_EQUALITY;
+			ret->u.equality.attr = attr;
+			ret->u.equality.value = ldb_binary_decode(ret, value);
+			if (ret->u.equality.value.data == NULL) {
+				talloc_free(ret);
+				return NULL;
+			}
+			talloc_free(value);
+			break;
+
+		case LDB_OP_GREATER:
+			ret->operation = LDB_OP_GREATER;
+			ret->u.comparison.attr = attr;
+			ret->u.comparison.value = ldb_binary_decode(ret, value);
+			if (ret->u.comparison.value.data == NULL) {
+				talloc_free(ret);
+				return NULL;
+			}
+			talloc_free(value);
+			break;
+
+		case LDB_OP_LESS:
+			ret->operation = LDB_OP_LESS;
+			ret->u.comparison.attr = attr;
+			ret->u.comparison.value = ldb_binary_decode(ret, value);
+			if (ret->u.comparison.value.data == NULL) {
+				talloc_free(ret);
+				return NULL;
+			}
+			talloc_free(value);
+			break;
+
+		case LDB_OP_APPROX:
+			ret->operation = LDB_OP_APPROX;
+			ret->u.comparison.attr = attr;
+			ret->u.comparison.value = ldb_binary_decode(ret, value);
+			if (ret->u.comparison.value.data == NULL) {
+				talloc_free(ret);
+				return NULL;
+			}
+			talloc_free(value);
+			break;
+
+		case LDB_OP_EXTENDED:
+
+			ret = ldb_parse_extended(ret, attr, value);
+			break;
+
+		default:
+			talloc_free(ret);
+			return NULL;
+	}
+
+	return ret;
+}
+
+
+/*
+  parse a filterlist
+  <and> ::= '&' <filterlist>
+  <or> ::= '|' <filterlist>
+  <filterlist> ::= <filter> | <filter> <filterlist>
+*/
+static struct ldb_parse_tree *ldb_parse_filterlist(
+	TALLOC_CTX *mem_ctx,
+	const char **s,
+	unsigned depth,
+	unsigned max_depth)
+{
+	struct ldb_parse_tree *ret, *next;
+	enum ldb_parse_op op;
+	const char *p = *s;
+
+	switch (*p) {
+		case '&':
+			op = LDB_OP_AND;
+			break;
+		case '|':
+			op = LDB_OP_OR;
+			break;
+		default:
+			return NULL;
+	}
+	p++;
+
+	while (isspace((unsigned char)*p)) p++;
+
+	ret = talloc(mem_ctx, struct ldb_parse_tree);
+	if (!ret) {
+		errno = ENOMEM;
+		return NULL;
+	}
+
+	ret->operation = op;
+	ret->u.list.num_elements = 1;
+	ret->u.list.elements = talloc(ret, struct ldb_parse_tree *);
+	if (!ret->u.list.elements) {
+		errno = ENOMEM;
+		talloc_free(ret);
+		return NULL;
+	}
+
+	ret->u.list.elements[0] =
+		ldb_parse_filter(ret->u.list.elements, &p, depth, max_depth);
+	if (!ret->u.list.elements[0]) {
+		talloc_free(ret);
+		return NULL;
+	}
+
+	while (isspace((unsigned char)*p)) p++;
+
+	while (*p) {
+		struct ldb_parse_tree **e;
+		if (*p == ')') {
+			break;
+		}
+
+		next = ldb_parse_filter(
+			ret->u.list.elements, &p, depth, max_depth);
+		if (next == NULL) {
+			/* an invalid filter element */
+			talloc_free(ret);
+			return NULL;
+		}
+		e = talloc_realloc(ret, ret->u.list.elements, 
+				     struct ldb_parse_tree *, 
+				     ret->u.list.num_elements + 1);
+		if (!e) {
+			errno = ENOMEM;
+			talloc_free(ret);
+			return NULL;
+		}
+		ret->u.list.elements = e;
+		ret->u.list.elements[ret->u.list.num_elements] = next;
+		ret->u.list.num_elements++;
+		while (isspace((unsigned char)*p)) p++;
+	}
+
+	*s = p;
+
+	return ret;
+}
+
+
+/*
+  <not> ::= '!' <filter>
+*/
+static struct ldb_parse_tree *ldb_parse_not(
+	TALLOC_CTX *mem_ctx,
+	const char **s,
+	unsigned depth,
+	unsigned max_depth)
+{
+	struct ldb_parse_tree *ret;
+	const char *p = *s;
+
+	if (*p != '!') {
+		return NULL;
+	}
+	p++;
+
+	ret = talloc(mem_ctx, struct ldb_parse_tree);
+	if (!ret) {
+		errno = ENOMEM;
+		return NULL;
+	}
+
+	ret->operation = LDB_OP_NOT;
+	ret->u.isnot.child = ldb_parse_filter(ret, &p, depth, max_depth);
+	if (!ret->u.isnot.child) {
+		talloc_free(ret);
+		return NULL;
+	}
+
+	*s = p;
+
+	return ret;
+}
+
+/*
+  parse a filtercomp
+  <filtercomp> ::= <and> | <or> | <not> | <simple>
+*/
+static struct ldb_parse_tree *ldb_parse_filtercomp(
+	TALLOC_CTX *mem_ctx,
+	const char **s,
+	unsigned depth,
+	unsigned max_depth)
+{
+	struct ldb_parse_tree *ret;
+	const char *p = *s;
+
+	while (isspace((unsigned char)*p)) p++;
+
+	switch (*p) {
+	case '&':
+		ret = ldb_parse_filterlist(mem_ctx, &p, depth, max_depth);
+		break;
+
+	case '|':
+		ret = ldb_parse_filterlist(mem_ctx, &p, depth, max_depth);
+		break;
+
+	case '!':
+		ret = ldb_parse_not(mem_ctx, &p, depth, max_depth);
+		break;
+
+	case '(':
+	case ')':
+		return NULL;
+
+	default:
+		ret = ldb_parse_simple(mem_ctx, &p);
+
+	}
+
+	*s = p;
+	return ret;
+}
+
+/*
+  <filter> ::= '(' <filtercomp> ')'
+*/
+static struct ldb_parse_tree *ldb_parse_filter(
+	TALLOC_CTX *mem_ctx,
+	const char **s,
+	unsigned depth,
+	unsigned max_depth)
+{
+	struct ldb_parse_tree *ret;
+	const char *p = *s;
+
+	/*
+	 * Check the depth of the parse tree, and reject the input if
+	 * max_depth exceeded. This avoids stack overflow
+	 * issues.
+	 */
+	if (depth > max_depth) {
+		return NULL;
+	}
+	depth++;
+
+	if (*p != '(') {
+		return NULL;
+	}
+	p++;
+
+	ret = ldb_parse_filtercomp(mem_ctx, &p, depth, max_depth);
+
+	if (*p != ')') {
+		return NULL;
+	}
+	p++;
+
+	while (isspace((unsigned char)*p)) {
+		p++;
+	}
+
+	*s = p;
+
+	return ret;
+}
+
+
+/*
+  main parser entry point. Takes a search string and returns a parse tree
+
+  expression ::= <simple> | <filter>
+*/
+struct ldb_parse_tree *ldb_parse_tree(TALLOC_CTX *mem_ctx, const char *s)
+{
+	unsigned depth = 0;
+
+	while (s && isspace((unsigned char)*s)) s++;
+
+	if (s == NULL || *s == 0) {
+		s = "(|(objectClass=*)(distinguishedName=*))";
+	}
+
+	if (*s == '(') {
+		return ldb_parse_filter(
+			mem_ctx, &s, depth, LDB_MAX_PARSE_TREE_DEPTH);
+	}
+
+	return ldb_parse_simple(mem_ctx, &s);
+}
+
+
+/*
+  construct a ldap parse filter given a parse tree
+*/
+char *ldb_filter_from_tree(TALLOC_CTX *mem_ctx, const struct ldb_parse_tree *tree)
+{
+	char *s, *s2, *ret;
+	unsigned int i;
+
+	if (tree == NULL) {
+		return NULL;
+	}
+
+	switch (tree->operation) {
+	case LDB_OP_AND:
+	case LDB_OP_OR:
+		ret = talloc_asprintf(mem_ctx, "(%c", tree->operation==LDB_OP_AND?'&':'|');
+		if (ret == NULL) return NULL;
+		for (i=0;i<tree->u.list.num_elements;i++) {
+			s = ldb_filter_from_tree(mem_ctx, tree->u.list.elements[i]);
+			if (s == NULL) {
+				talloc_free(ret);
+				return NULL;
+			}
+			s2 = talloc_asprintf_append(ret, "%s", s);
+			talloc_free(s);
+			if (s2 == NULL) {
+				talloc_free(ret);
+				return NULL;
+			}
+			ret = s2;
+		}
+		s = talloc_asprintf_append(ret, ")");
+		if (s == NULL) {
+			talloc_free(ret);
+			return NULL;
+		}
+		return s;
+	case LDB_OP_NOT:
+		s = ldb_filter_from_tree(mem_ctx, tree->u.isnot.child);
+		if (s == NULL) return NULL;
+
+		ret = talloc_asprintf(mem_ctx, "(!%s)", s);
+		talloc_free(s);
+		return ret;
+	case LDB_OP_EQUALITY:
+		s = ldb_binary_encode(mem_ctx, tree->u.equality.value);
+		if (s == NULL) return NULL;
+		ret = talloc_asprintf(mem_ctx, "(%s=%s)", 
+				      tree->u.equality.attr, s);
+		talloc_free(s);
+		return ret;
+	case LDB_OP_SUBSTRING:
+		ret = talloc_asprintf(mem_ctx, "(%s=%s", tree->u.substring.attr,
+				      tree->u.substring.start_with_wildcard?"*":"");
+		if (ret == NULL) return NULL;
+		for (i = 0; tree->u.substring.chunks && tree->u.substring.chunks[i]; i++) {
+			s2 = ldb_binary_encode(mem_ctx, *(tree->u.substring.chunks[i]));
+			if (s2 == NULL) {
+				talloc_free(ret);
+				return NULL;
+			}
+			if (tree->u.substring.chunks[i+1] ||
+			    tree->u.substring.end_with_wildcard) {
+				s = talloc_asprintf_append(ret, "%s*", s2);
+			} else {
+				s = talloc_asprintf_append(ret, "%s", s2);
+			}
+			if (s == NULL) {
+				talloc_free(ret);
+				return NULL;
+			}
+			ret = s;
+		}
+		s = talloc_asprintf_append(ret, ")");
+		if (s == NULL) {
+			talloc_free(ret);
+			return NULL;
+		}
+		ret = s;
+		return ret;
+	case LDB_OP_GREATER:
+		s = ldb_binary_encode(mem_ctx, tree->u.equality.value);
+		if (s == NULL) return NULL;
+		ret = talloc_asprintf(mem_ctx, "(%s>=%s)", 
+				      tree->u.equality.attr, s);
+		talloc_free(s);
+		return ret;
+	case LDB_OP_LESS:
+		s = ldb_binary_encode(mem_ctx, tree->u.equality.value);
+		if (s == NULL) return NULL;
+		ret = talloc_asprintf(mem_ctx, "(%s<=%s)", 
+				      tree->u.equality.attr, s);
+		talloc_free(s);
+		return ret;
+	case LDB_OP_PRESENT:
+		ret = talloc_asprintf(mem_ctx, "(%s=*)", tree->u.present.attr);
+		return ret;
+	case LDB_OP_APPROX:
+		s = ldb_binary_encode(mem_ctx, tree->u.equality.value);
+		if (s == NULL) return NULL;
+		ret = talloc_asprintf(mem_ctx, "(%s~=%s)", 
+				      tree->u.equality.attr, s);
+		talloc_free(s);
+		return ret;
+	case LDB_OP_EXTENDED:
+		s = ldb_binary_encode(mem_ctx, tree->u.extended.value);
+		if (s == NULL) return NULL;
+		ret = talloc_asprintf(mem_ctx, "(%s%s%s%s:=%s)", 
+				      tree->u.extended.attr?tree->u.extended.attr:"", 
+				      tree->u.extended.dnAttributes?":dn":"",
+				      tree->u.extended.rule_id?":":"", 
+				      tree->u.extended.rule_id?tree->u.extended.rule_id:"", 
+				      s);
+		talloc_free(s);
+		return ret;
+	}
+	
+	return NULL;
+}
+
+
+/*
+  walk a parse tree, calling the provided callback on each node
+*/
+int ldb_parse_tree_walk(struct ldb_parse_tree *tree,
+			int (*callback)(struct ldb_parse_tree *tree, void *),
+			void *private_context)
+{
+	unsigned int i;
+	int ret;
+
+	ret = callback(tree, private_context);
+	if (ret != LDB_SUCCESS) {
+		return ret;
+	}
+
+	switch (tree->operation) {
+	case LDB_OP_AND:
+	case LDB_OP_OR:
+		for (i=0;i<tree->u.list.num_elements;i++) {
+			ret = ldb_parse_tree_walk(tree->u.list.elements[i], callback, private_context);
+			if (ret != LDB_SUCCESS) {
+				return ret;
+			}
+		}
+		break;
+	case LDB_OP_NOT:
+		ret = ldb_parse_tree_walk(tree->u.isnot.child, callback, private_context);
+		if (ret != LDB_SUCCESS) {
+			return ret;
+		}
+		break;
+	case LDB_OP_EQUALITY:
+	case LDB_OP_GREATER:
+	case LDB_OP_LESS:
+	case LDB_OP_APPROX:
+	case LDB_OP_SUBSTRING:
+	case LDB_OP_PRESENT:
+	case LDB_OP_EXTENDED:
+		break;
+	}
+	return LDB_SUCCESS;
+}
+
+struct parse_tree_attr_replace_ctx {
+	const char *attr;
+	const char *replace;
+};
+
+/*
+  callback for ldb_parse_tree_attr_replace()
+ */
+static int parse_tree_attr_replace(struct ldb_parse_tree *tree, void *private_context)
+{
+	struct parse_tree_attr_replace_ctx *ctx = private_context;
+	switch (tree->operation) {
+	case LDB_OP_EQUALITY:
+	case LDB_OP_GREATER:
+	case LDB_OP_LESS:
+	case LDB_OP_APPROX:
+		if (ldb_attr_cmp(tree->u.equality.attr, ctx->attr) == 0) {
+			tree->u.equality.attr = ctx->replace;
+		}
+		break;
+	case LDB_OP_SUBSTRING:
+		if (ldb_attr_cmp(tree->u.substring.attr, ctx->attr) == 0) {
+			tree->u.substring.attr = ctx->replace;
+		}
+		break;
+	case LDB_OP_PRESENT:
+		if (ldb_attr_cmp(tree->u.present.attr, ctx->attr) == 0) {
+			tree->u.present.attr = ctx->replace;
+		}
+		break;
+	case LDB_OP_EXTENDED:
+		if (tree->u.extended.attr &&
+		    ldb_attr_cmp(tree->u.extended.attr, ctx->attr) == 0) {
+			tree->u.extended.attr = ctx->replace;
+		}
+		break;
+	default:
+		break;
+	}
+	return LDB_SUCCESS;
+}
+
+/*
+  replace any occurrences of an attribute name in the parse tree with a
+  new name
+*/
+void ldb_parse_tree_attr_replace(struct ldb_parse_tree *tree,
+				 const char *attr,
+				 const char *replace)
+{
+	struct parse_tree_attr_replace_ctx ctx;
+
+	ctx.attr    = attr;
+	ctx.replace = replace;
+
+	ldb_parse_tree_walk(tree, parse_tree_attr_replace, &ctx);
+}
+
+/*
+  shallow copy a tree - copying only the elements array so that the caller
+  can safely add new elements without changing the message
+*/
+struct ldb_parse_tree *ldb_parse_tree_copy_shallow(TALLOC_CTX *mem_ctx,
+						   const struct ldb_parse_tree *ot)
+{
+	unsigned int i;
+	struct ldb_parse_tree *nt;
+
+	nt = talloc(mem_ctx, struct ldb_parse_tree);
+	if (!nt) {
+		return NULL;
+	}
+
+	*nt = *ot;
+
+	switch (ot->operation) {
+	case LDB_OP_AND:
+	case LDB_OP_OR:
+		nt->u.list.elements = talloc_array(nt, struct ldb_parse_tree *,
+						   ot->u.list.num_elements);
+		if (!nt->u.list.elements) {
+			talloc_free(nt);
+			return NULL;
+		}
+
+		for (i=0;i<ot->u.list.num_elements;i++) {
+			nt->u.list.elements[i] =
+				ldb_parse_tree_copy_shallow(nt->u.list.elements,
+						ot->u.list.elements[i]);
+			if (!nt->u.list.elements[i]) {
+				talloc_free(nt);
+				return NULL;
+			}
+		}
+		break;
+	case LDB_OP_NOT:
+		nt->u.isnot.child = ldb_parse_tree_copy_shallow(nt,
+							ot->u.isnot.child);
+		if (!nt->u.isnot.child) {
+			talloc_free(nt);
+			return NULL;
+		}
+		break;
+	case LDB_OP_EQUALITY:
+	case LDB_OP_GREATER:
+	case LDB_OP_LESS:
+	case LDB_OP_APPROX:
+	case LDB_OP_SUBSTRING:
+	case LDB_OP_PRESENT:
+	case LDB_OP_EXTENDED:
+		break;
+	}
+
+	return nt;
+}
+
+/* Get the attribute (if any) associated with the top node of a parse tree. */
+const char *ldb_parse_tree_get_attr(const struct ldb_parse_tree *tree)
+{
+	switch (tree->operation) {
+	case LDB_OP_AND:
+	case LDB_OP_OR:
+	case LDB_OP_NOT:
+		return NULL;
+	case LDB_OP_EQUALITY:
+		return tree->u.equality.attr;
+	case LDB_OP_SUBSTRING:
+		return tree->u.substring.attr;
+	case LDB_OP_GREATER:
+	case LDB_OP_LESS:
+	case LDB_OP_APPROX:
+		return tree->u.comparison.attr;
+	case LDB_OP_PRESENT:
+		return tree->u.present.attr;
+	case LDB_OP_EXTENDED:
+		return tree->u.extended.attr;
+	}
+
+	return NULL;
+}
diff --git a/lib/ldb/common/ldb_utf8.c b/lib/ldb/common/ldb_utf8.c
new file mode 100644
index 0000000..55d8f90
--- /dev/null
+++ b/lib/ldb/common/ldb_utf8.c
@@ -0,0 +1,136 @@
+/* 
+   ldb database library
+
+   Copyright (C) Andrew Tridgell  2004
+
+     ** NOTE! The following LGPL license applies to the ldb
+     ** library. This does NOT imply that all of Samba is released
+     ** under the LGPL
+   
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+/*
+ *  Name: ldb
+ *
+ *  Component: ldb utf8 handling
+ *
+ *  Description: case folding and case comparison for UTF8 strings
+ *
+ *  Author: Andrew Tridgell
+ */
+
+#include "ldb_private.h"
+#include "system/locale.h"
+
+
+/*
+  this allow the user to pass in a caseless comparison
+  function to handle utf8 caseless comparisons
+ */
+void ldb_set_utf8_fns(struct ldb_context *ldb,
+		      void *context,
+		      char *(*casefold)(void *, void *, const char *, size_t))
+{
+	if (context)
+		ldb->utf8_fns.context = context;
+	if (casefold)
+		ldb->utf8_fns.casefold = casefold;
+}
+
+/*
+  a simple case folding function
+  NOTE: does not handle UTF8
+*/
+char *ldb_casefold_default(void *context, TALLOC_CTX *mem_ctx, const char *s, size_t n)
+{
+	size_t i;
+	char *ret = talloc_strndup(mem_ctx, s, n);
+	if (!s) {
+		errno = ENOMEM;
+		return NULL;
+	}
+	for (i=0;ret[i];i++) {
+		ret[i] = toupper((unsigned char)ret[i]);
+	}
+	return ret;
+}
+
+void ldb_set_utf8_default(struct ldb_context *ldb)
+{
+	ldb_set_utf8_fns(ldb, NULL, ldb_casefold_default);
+}
+
+char *ldb_casefold(struct ldb_context *ldb, TALLOC_CTX *mem_ctx, const char *s, size_t n)
+{
+	return ldb->utf8_fns.casefold(ldb->utf8_fns.context, mem_ctx, s, n);
+}
+
+/*
+  check the attribute name is valid according to rfc2251
+  returns 1 if the name is ok
+ */
+
+int ldb_valid_attr_name(const char *s)
+{
+	size_t i;
+
+	if (!s || !s[0])
+		return 0;
+
+	/* handle special ldb_tdb wildcard */
+	if (strcmp(s, "*") == 0) return 1;
+
+	for (i = 0; s[i]; i++) {
+		if (! isascii(s[i])) {
+			return 0;
+		}
+		if (i == 0) { /* first char must be an alpha (or our special '@' identifier) */
+			if (! (isalpha(s[i]) || (s[i] == '@'))) {
+				return 0;
+			}
+		} else {
+			if (! (isalnum(s[i]) || (s[i] == '-'))) {
+				return 0;
+			}
+		}
+	}
+	return 1;
+}
+
+char *ldb_attr_casefold(TALLOC_CTX *mem_ctx, const char *s)
+{
+	size_t i;
+	char *ret = talloc_strdup(mem_ctx, s);
+	if (!ret) {
+		errno = ENOMEM;
+		return NULL;
+	}
+	for (i = 0; ret[i]; i++) {
+		ret[i] = toupper((unsigned char)ret[i]);
+	}
+	return ret;
+}
+
+/*
+  we accept either 'dn' or 'distinguishedName' for a distinguishedName
+*/
+int ldb_attr_dn(const char *attr)
+{
+	if (ldb_attr_cmp(attr, "dn") == 0 ||
+	    ldb_attr_cmp(attr, "distinguishedName") == 0) {
+		return 0;
+	}
+	return -1;
+}
diff --git a/lib/ldb/common/qsort.c b/lib/ldb/common/qsort.c
new file mode 100644
index 0000000..012aaf3
--- /dev/null
+++ b/lib/ldb/common/qsort.c
@@ -0,0 +1,251 @@
+/* Copyright (C) 1991,1992,1996,1997,1999,2004 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Written by Douglas C. Schmidt (schmidt@ics.uci.edu).
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see <http://www.gnu.org/licenses/>. */
+
+/* If you consider tuning this algorithm, you should consult first:
+   Engineering a sort function; Jon Bentley and M. Douglas McIlroy;
+   Software - Practice and Experience; Vol. 23 (11), 1249-1265, 1993.  */
+
+/* Modified to be used in samba4 by
+ * Simo Sorce <idra@samba.org>		2005
+ */
+
+#include "ldb_private.h"
+
+/* Byte-wise swap two items of size SIZE. */
+#define SWAP(a, b, size)						      \
+  do									      \
+    {									      \
+      register size_t __size = (size);					      \
+      register char *__a = (a), *__b = (b);				      \
+      do								      \
+	{								      \
+	  char __tmp = *__a;						      \
+	  *__a++ = *__b;						      \
+	  *__b++ = __tmp;						      \
+	} while (--__size > 0);						      \
+    } while (0)
+
+/* Discontinue quicksort algorithm when partition gets below this size.
+   This particular magic number was chosen to work best on a Sun 4/260. */
+#define MAX_THRESH 4
+
+/* Stack node declarations used to store unfulfilled partition obligations. */
+typedef struct
+  {
+    char *lo;
+    char *hi;
+  } stack_node;
+
+/* The next 4 #defines implement a very fast in-line stack abstraction. */
+/* The stack needs log (total_elements) entries (we could even subtract
+   log(MAX_THRESH)).  Since total_elements has type size_t, we get as
+   upper bound for log (total_elements):
+   bits per byte (CHAR_BIT) * sizeof(size_t).  */
+#ifndef CHAR_BIT
+#define CHAR_BIT 8
+#endif
+#define STACK_SIZE	(CHAR_BIT * sizeof(size_t))
+#define PUSH(low, high) ((void) ((stack[i].lo = (low)), (stack[i].hi = (high)), i++))
+#define POP(low, high)  ((void) (i--, (low = stack[i].lo), (high = stack[i].hi)))
+
+
+/* Order size using quicksort.  This implementation incorporates
+   four optimizations discussed in Sedgewick:
+
+   1. Non-recursive, using an explicit stack of pointer that store the
+      next array partition to sort.  To save time, this maximum amount
+      of space required to store an array of SIZE_MAX is allocated on the
+      stack.  Assuming a 32-bit (64 bit) integer for size_t, this needs
+      only 32 * sizeof(stack_node) == 256 bytes (for 64 bit: 1024 bytes).
+      Pretty cheap, actually.
+
+   2. Chose the pivot element using a median-of-three decision tree.
+      This reduces the probability of selecting a bad pivot value and
+      eliminates certain extraneous comparisons.
+
+   3. Only quicksorts TOTAL_ELEMS / MAX_THRESH partitions, leaving
+      insertion sort to order the MAX_THRESH items within each partition.
+      This is a big win, since insertion sort is faster for small, mostly
+      sorted array segments.
+
+   4. The larger of the two sub-partitions is always pushed onto the
+      stack first, with the algorithm then concentrating on the
+      smaller partition.  This *guarantees* no more than log (total_elems)
+      stack size is needed (actually O(1) in this case)!  */
+
+void ldb_qsort (void *const pbase, size_t total_elems, size_t size,
+		void *opaque, ldb_qsort_cmp_fn_t cmp)
+{
+  register char *base_ptr = (char *) pbase;
+
+  const size_t max_thresh = MAX_THRESH * size;
+
+  if (total_elems == 0)
+    /* Avoid lossage with unsigned arithmetic below.  */
+    return;
+
+  if (total_elems > MAX_THRESH)
+    {
+      char *lo = base_ptr;
+      char *hi = &lo[size * (total_elems - 1)];
+      stack_node stack[STACK_SIZE];
+      size_t i = 0;
+
+      PUSH (NULL, NULL);
+
+      do
+        {
+          char *left_ptr;
+          char *right_ptr;
+
+	  /* Select median value from among LO, MID, and HI. Rearrange
+	     LO and HI so the three values are sorted. This lowers the
+	     probability of picking a pathological pivot value and
+	     skips a comparison for both the LEFT_PTR and RIGHT_PTR in
+	     the while loops. */
+
+	  char *mid = lo + size * ((hi - lo) / size >> 1);
+
+	  if ((*cmp) ((void *) mid, (void *) lo, opaque) < 0)
+	    SWAP (mid, lo, size);
+	  if ((*cmp) ((void *) hi, (void *) mid, opaque) < 0)
+	    SWAP (mid, hi, size);
+	  else
+	    goto jump_over;
+	  if ((*cmp) ((void *) mid, (void *) lo, opaque) < 0)
+	    SWAP (mid, lo, size);
+	jump_over:;
+
+	  left_ptr  = lo + size;
+	  right_ptr = hi - size;
+
+	  /* Here's the famous ``collapse the walls'' section of quicksort.
+	     Gotta like those tight inner loops!  They are the main reason
+	     that this algorithm runs much faster than others. */
+	  do
+	    {
+	      while ((*cmp) ((void *) left_ptr, (void *) mid, opaque) < 0)
+		left_ptr += size;
+
+	      while ((*cmp) ((void *) mid, (void *) right_ptr, opaque) < 0)
+		right_ptr -= size;
+
+	      if (left_ptr < right_ptr)
+		{
+		  SWAP (left_ptr, right_ptr, size);
+		  if (mid == left_ptr)
+		    mid = right_ptr;
+		  else if (mid == right_ptr)
+		    mid = left_ptr;
+		  left_ptr += size;
+		  right_ptr -= size;
+		}
+	      else if (left_ptr == right_ptr)
+		{
+		  left_ptr += size;
+		  right_ptr -= size;
+		  break;
+		}
+	    }
+	  while (left_ptr <= right_ptr);
+
+          /* Set up pointers for next iteration.  First determine whether
+             left and right partitions are below the threshold size.  If so,
+             ignore one or both.  Otherwise, push the larger partition's
+             bounds on the stack and continue sorting the smaller one. */
+
+          if ((size_t) (right_ptr - lo) <= max_thresh)
+            {
+              if ((size_t) (hi - left_ptr) <= max_thresh)
+		/* Ignore both small partitions. */
+                POP (lo, hi);
+              else
+		/* Ignore small left partition. */
+                lo = left_ptr;
+            }
+          else if ((size_t) (hi - left_ptr) <= max_thresh)
+	    /* Ignore small right partition. */
+            hi = right_ptr;
+          else if ((right_ptr - lo) > (hi - left_ptr))
+            {
+	      /* Push larger left partition indices. */
+              PUSH (lo, right_ptr);
+              lo = left_ptr;
+            }
+          else
+            {
+	      /* Push larger right partition indices. */
+              PUSH (left_ptr, hi);
+              hi = right_ptr;
+            }
+        }
+      while (i > 0 && i < STACK_SIZE);
+    }
+
+  /* Once the BASE_PTR array is partially sorted by quicksort the rest
+     is completely sorted using insertion sort, since this is efficient
+     for partitions below MAX_THRESH size. BASE_PTR points to the beginning
+     of the array to sort, and END_PTR points at the very last element in
+     the array (*not* one beyond it!). */
+
+#define min(x, y) ((x) < (y) ? (x) : (y))
+
+  {
+    char *const end_ptr = &base_ptr[size * (total_elems - 1)];
+    char *tmp_ptr = base_ptr;
+    char *thresh = min(end_ptr, base_ptr + max_thresh);
+    register char *run_ptr;
+
+    /* Find smallest element in first threshold and place it at the
+       array's beginning.  This is the smallest array element,
+       and the operation speeds up insertion sort's inner loop. */
+
+    for (run_ptr = tmp_ptr + size; run_ptr <= thresh; run_ptr += size)
+      if ((*cmp) ((void *) run_ptr, (void *) tmp_ptr, opaque) < 0)
+        tmp_ptr = run_ptr;
+
+    if (tmp_ptr != base_ptr)
+      SWAP (tmp_ptr, base_ptr, size);
+
+    /* Insertion sort, running from left-hand-side up to right-hand-side.  */
+
+    run_ptr = base_ptr + size;
+    while ((run_ptr += size) <= end_ptr)
+      {
+	tmp_ptr = run_ptr - size;
+	while ((*cmp) ((void *) run_ptr, (void *) tmp_ptr, opaque) < 0)
+	  tmp_ptr -= size;
+
+	tmp_ptr += size;
+        if (tmp_ptr != run_ptr)
+          {
+            char *trav;
+
+	    trav = run_ptr + size;
+	    while (--trav >= run_ptr)
+              {
+                char c = *trav;
+                char *hi, *lo;
+
+                for (hi = lo = trav; (lo -= size) >= tmp_ptr; hi = lo)
+                  *hi = *lo;
+                *hi = c;
+              }
+          }
+      }
+  }
+}