summaryrefslogtreecommitdiffstats
path: root/doc/devel
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--doc/devel/OIDs119
-rw-r--r--doc/devel/README9
-rw-r--r--doc/devel/args64
-rw-r--r--doc/devel/lloadd/design.md282
-rw-r--r--doc/devel/template.c26
-rw-r--r--doc/devel/todo67
-rw-r--r--doc/devel/toolargs31
-rw-r--r--doc/devel/utfconv.txt291
-rw-r--r--doc/devel/variadic_debug/03-libldap_Debug.cocci70
-rw-r--r--doc/devel/variadic_debug/04-variadic.cocci165
-rw-r--r--doc/devel/variadic_debug/07-shortcut.cocci216
-rw-r--r--doc/devel/variadic_debug/09-merge.cocci147
-rw-r--r--doc/devel/variadic_debug/README39
-rw-r--r--doc/devel/variadic_debug/equivalence.iso12
-rw-r--r--doc/devel/variadic_debug/macros.h23
-rwxr-xr-xdoc/devel/variadic_debug/script.sh73
16 files changed, 1634 insertions, 0 deletions
diff --git a/doc/devel/OIDs b/doc/devel/OIDs
new file mode 100644
index 0000000..af2bf88
--- /dev/null
+++ b/doc/devel/OIDs
@@ -0,0 +1,119 @@
+OpenLDAProot 1.3.6.1.4.1.4203
+
+OpenLDAP OpenLDAProot:1
+
+OpenLDAPsyntax OpenLDAP:1
+ authPasswordSyntax OpenLDAPsyntax:2
+
+OpenLDAPmatchingrule OpenLDAP:2
+ authPasswordExactMatch OpenLDAPmatchingrule:2
+ authPasswordMatch OpenLDAPmatchingrule:3
+
+OpenLDAPattributeType OpenLDAP:3
+ supportedAuthPasswordSchemas OpenLDAPattributeType:3
+ authPassword OpenLDAPattributeType:4
+ supportedFeatures OpenLDAPattributeType:5
+
+OpenLDAPobjectClass OpenLDAP:4
+ OpenLDAPorg OpenLDAPObjectClass:3
+ OpenLDAPou OpenLDAPObjectClass:4
+ OpenLDAPperson OpenLDAPObjectClass:5
+ OpenLDAPdisplayableObject OpenLDAPObjectClass:6
+ authPasswordObject OpenLDAPobjectClass:7
+
+OpenLDAPfeatures OpenLDAP:5
+ allOperationalAttrs OpenLDAPfeatures:1
+ OC AD lists OpenLDAPfeatures:2
+ TrueFalseFilters OpenLDAPfeatures:3
+ languageTagOptions OpenLDAPfeatures:4
+ languageRangetags OpenLDAPfeatures:5
+
+Syncrepl OpenLDAP:9
+
+OpenLDAPcontrol OpenLDAP:10
+ SubentriesControl OpenLDAPcontrol:1
+
+OpenLDAPexop OpenLDAP:11
+ passwordModify OpenLDAPexop:1
+ whoAmI OpenLDAPexop:3
+
+OpenLDAPinternal OpenLDAP:12
+ OpenLDAPtesting OpenLDAPinternal:1
+ OpenLDAPconfig OpenLDAPinternal:2
+
+
+OpenLDAPexperimental OpenLDAProot:666
+
+ExperimentalAttr OpenLDAPexperimental:1
+ OpenLDAPaci ExperimentalAttr:5
+ entryCSN ExperimentalAttr:7
+ authzTo ExperimentalAttr:8
+ authzFrom ExperimentalAttr:9
+ monitorContext ExperimentalAttr:10
+ superiorUUID ExperimentalAttr:11 check - is this dup of parentUUID?
+ namingCSN ExperimentalAttr:13
+ syncreplCookie ExperimentalAttr:23
+ contextCSN ExperimentalAttr:25
+ syncTimestamp ExperimentalAttr:26
+ lastmodDN ExperimentalAttr:28 (contrib/slapd-modules/lastmod)
+ lastmodType ExperimentalAttr:29
+ lastmodEnabled ExperimentalAttr:30
+ monitorAttrs ExperimentalAttr:55 (back-monitor)
+ entryExpireTimestamp ExperimentalAttr:57 (slapo-dds)
+ rdnValue ExperimentalAttr:58 (contrib/slapd-modules/samba4)
+ parentUUID ExperimentalAttr:59 (...samba4)
+ x509PrivateKey ExperimentalAttr:60
+
+
+ExperimentalSyntax OpenLDAPexperimental:2
+ ACIsyntax ExperimentalSyntax:1
+ authPassword ExperimentalSyntax:2 check - this was promoted to RFC3112
+ authz ExperimentalSyntax:7
+ privateKey ExperimentalSyntax:13
+
+ExperimentalObjectClass OpenLDAPexperimental:3
+ glue ExperimentalObjectClass:4
+ syncConsumerSubentry ExperimentalObjectClass:5
+ syncProviderSubentry ExperimentalObjectClass:6
+ lastmod ExperimentalObjectClass:13
+ monitorClasses ExperimentalObjectClass:16
+
+ExperimentalMatchingRule OpenLDAPexperimental:4
+ authPaswordMatch ExperimentalMatchingRule:1 check - this was promoted to RFC3112
+ ACImatch ExperimentalMatchingRule:2
+ direectoryStringApproxMatch ExperimentalMatchingRule:4
+ IA5stringApproxMatch ExperimentalMatchingRule:5
+ dnOneLevelMatch ExperimentalMatchingRule:8
+ dnSubtreeMatch ExperimentalMatchingRule:9
+ dnSubordinateMatch ExperimentalMatchingRule:10
+ dnSuperiorMatch ExperimentalMatchingRule:11
+ authzMatch ExperimentalMatchingRule:12
+ privateKeyMatch ExperimentalMatchingRule:13
+
+ExperimentalControl OpenLDAPexperimental:5
+ noop ExperimentalControl:2
+ noSubordinates ExperimentalControl:11
+ relax ExperimentalControl:12
+ slurp ExperimentalControl:13
+ valsort ExperimentalControl:14
+ deref ExperimentalControl:16
+ whatfailed ExperimentalControl:17
+ noopsrch ExperimentalControl:18
+
+ExperimentalExop OpenLDAPexperimental:6
+ verifyCredentials ExperimentalExop:5
+
+ExperimentalFeatures OpenLDAPexperimental:8
+ subordinateScope ExperimentalFeatures:1
+
+SelfContainedWorks OpenLDAPexperimental:11
+ CSNs SelfContainedWorks:2
+ chaining SelfContainedWorks:3
+ retcode SelfContainedWorks:4
+ accesslog SelfContainedWorks:5
+ distProc SelfContainedWorks:6
+ LDAP txns SelfContainedWorks:7 (replaced by 1.3.6.1.1.21 RFC 5805)
+ dyngroup SelfContainedWorks:8
+ proxyCache SelfContainedWorks:9
+ X509 PMI SelfContainedWorks:10
+ autoca SelfContainedWorks:11
diff --git a/doc/devel/README b/doc/devel/README
new file mode 100644
index 0000000..3a0cb3d
--- /dev/null
+++ b/doc/devel/README
@@ -0,0 +1,9 @@
+The OpenLDAP Developer's FAQ is available at:
+ http://www.openldap.org/faq/index.cgi?file=4
+
+Additional developer pages are at:
+ http://www.openldap.org/devel/
+
+
+---
+$OpenLDAP$
diff --git a/doc/devel/args b/doc/devel/args
new file mode 100644
index 0000000..c5aa02f
--- /dev/null
+++ b/doc/devel/args
@@ -0,0 +1,64 @@
+Tools ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz
+ldapcompare * DE**HI** MNOPQR UVWXYZ de *h*** *nop* vwxyz
+ldapdelete *CDE**HI** MNOPQR UVWXYZ cdef*h*** *nop* vwxyz
+ldapexop * D **HI** NO QR UVWXYZ de *h*** *nop vwxy
+ldapmodify *CDE**HI** MNOPQRS UVWXYZabcde *h*** *nop*r t vwxy
+ldapmodrdn *CDE**HI** MNOPQR UVWXYZ cdef*h*** *nop*rs vwxy
+ldappasswd A*CDE**HI** NO QRS UVWXYZa def*h*** * o * s vwxy
+ldapsearch A*CDE**HI**LMNOPQRSTUVWXYZab def*h***l*nop* stuvwxyz
+ldapurl * E**H ** S ab f*h*** * p* s
+ldapvc * DE**HI** NO QRS UVWXYZa cdef*h*** *nop* vwxy
+ldapwhoami * DE**HI** NO QR UVWXYZ def*h*** *nop* vwxy
+
+
+* reserved
+ BFGJgijmq01235789
+
+* General flags:
+ -C Chase Referrals
+ -D Bind DN
+ -E Tool-specific Extensions (e.g., -E <[!]oid[=options]>*)
+ -e General Extensions (e.g., -e <[!]oid[=options]>*)
+ -f file
+ -H URI
+ -P protocol version
+ -V version information
+ -W prompt for bind password
+ -d debug
+ -h host
+ -n no-op
+ -N no (SASLprep) normalization of simple bind password
+ -o general libldap options (plus ldif_wrap and nettimeout for backwards comp.)
+ -p port
+ -v verbose
+ -V version
+ -x simple bind
+ -y Bind password-file
+ -w Bind password
+
+Not used
+ -4 IPv4 only
+ -6 IPv6 only
+
+
+* LDAPv3 Only
+ -M ManageDSAIT
+ -Z StartTLS
+
+ -Y SASL Mechanism (defaults to "best")
+ -R SASL Realm (defaults to empty)
+ -O SASL Security Options (defaults to "noanonymous,noplain")
+ -U SASL Authentication Identity (defaults to USER)
+ -X SASL Authorization Identity (defaults to empty)
+
+ -I SASL interactive mode (default: automatic)
+ -Q SASL quiet mode (default: automatic)
+
+
+* LDAPv2+ Only (REMOVED)
+ -K LDAPv2 Kerberos Bind (Step 1 only)
+ -k LDAPv2 Kerberos Bind
+
+
+---
+$OpenLDAP$
diff --git a/doc/devel/lloadd/design.md b/doc/devel/lloadd/design.md
new file mode 100644
index 0000000..62fcd88
--- /dev/null
+++ b/doc/devel/lloadd/design.md
@@ -0,0 +1,282 @@
+TODO:
+- [ ] keep a global op in-flight counter? (might need locking)
+- [-] scheduling (who does what, more than one select thread? How does the proxy
+ work get distributed between threads?)
+- [ ] managing timeouts?
+- [X] outline locking policy: seems like there might be a lock inversion in the
+ design looming: when working with op, might need a lock on both client and
+ upstream but depending on where we started, we might want to start with
+ locking one, then other
+- [ ] how to deal with the balancer running out of fds? Especially when we hit
+ the limit, then lose an upstream connection and accept() a client, we
+ wouldn't be able to initiate a new one. A bit of a DoS... But probably not
+ a concern for Ericsson
+- [ ] non-Linux? No idea how anything other than poll works (moot if building a
+ libevent/libuv-based load balancer since they take care of that, except
+ edge-triggered I/O?)
+- [-] rootDSE? Controls and exops might have different semantics and need
+ binding to the same upstream connection.
+- [ ] Just piggybacking on OpenLDAP as a module? Would still need some updates
+ in the core and the module/subsystem would be a very invasive one. On the
+ other hand, allows to expose live configuration and monitoring over LDAP
+ over the current slapd listeners without re-inventing the wheel.
+
+
+Expecting to handle only LDAPv3
+
+terms:
+ server - configured target
+ upstream - a single connection to a server
+ client - an incoming connection
+
+To maintain fairness `G( requested => ( F( progressed | failed ) ) )`, use
+queues and put timeouts in
+
+Runtime organisation
+------
+- main thread with its own event base handling signals
+- one thread (later possibly more) listening on the rendezvous sockets, handing
+ the new sockets to worker threads
+- n worker threads dealing with client and server I/O (dispatching actual work
+ to the thread pool most likely)
+- a thread pool to handle actual work
+
+Operational behaviour
+------
+
+- client read -> upstream write:
+ - client read:
+ - if TLS_SETUP, keep processing, set state back when finished and note that
+ we're under TLS
+ - ber_get_next(), if we don't have a tag, finished (unless we have true
+ edge-triggered I/O, also put the fd back into the ones we're waiting for)
+ - peek at op tag:
+ - unbind:
+ - with a single lock, mark all pending ops in upstreams abandoned, clear
+ client link (would it be fast enough if we remove them from upstream
+ map instead?)
+ - locked per op:
+ - remove op from upstream map
+ - check upstream is not write-suspended, if it is ...
+ - try to write the abandon op to upstream, suspend upstream if not
+ fully sent
+ - remove op from client map (how if we're in avl_apply?, another pass?)
+ - would be nice if we could wipe the complete client map then, otherwise
+ we need to queue it to have it freed when all abandons get passed onto
+ the upstream (just dropping them might put extra strain on upstreams,
+ will probably have a queue on each client/upstream anyway, not just a
+ single Ber)
+ - bind:
+ - check mechanism is not EXTERNAL (or implement it)
+ - abandon existing ops (see unbind)
+ - set state to BINDING, put DN into authzid
+ - pick upstream, create PDU and sent
+ - abandon:
+ - find op, mark for abandon, send to appropriate upstream
+ - Exop:
+ - check not BINDING (unless it's a cancel?)
+ - check OID:
+ - STARTTLS:
+ - check we don't have TLS yet
+ - abandon all
+ - set state to TLS_SETUP
+ - send the hello
+ - VC(?):
+ - similar to bind except for the abandons/state change
+ - other:
+ - check not BINDING
+ - pick an upstream
+ - create a PDU, send (marking upstream suspended if not written in full)
+ - check if should read again (keep a counter of number of times to read
+ off a connection in a single pass so that we maintain fairness)
+ - if read enough requests and can still read, re-queue ourselves (if we
+ don't have true edge-triggered I/O, we can just register the fd again)
+ - upstream write (only when suspended):
+ - flush the current BER
+ - there shouldn't be anything else?
+- upstream read -> client write:
+ - upstream read:
+ - ber_get_next(), if we don't have a tag, finished (unless we have true
+ edge-triggered I/O, also put the fd back into the ones we're waiting for)
+ - when we get it, peek at msgid, resolve client connection, lock, check:
+ - if unsolicited, handle as close (and mark connection closing)
+ - if op is abandoned or does not exist, drop PDU and op, update counters
+ - if client backlogged, suspend upstream, register callback to unsuspend
+ (on progress when writing to client or abandon from client (connection
+ death, abandon proper, ...))
+ - reconstruct final PDU, write BER to client, if did not write fully,
+ suspend client
+ - if a final response, decrement operation counts on upstream and client
+ - check if should read again (keep a counter of number of responses to read
+ off a connection in a single pass so that we don't starve any?)
+ - client write ready (only checked for when suspended):
+ - write the rest of pending BER if any
+ - on successful write, pick all pending ops that need failure response, push
+ to client (are there any controls that need to be present in response even
+ in the case of failure?, what to do with them?)
+ - on successfully flushing them, walk through suspended upstreams, picking
+ the pending PDU (unsuspending the upstream) and writing, if PDU flushed
+ successfully, pick next upstream
+ - if we successfully flushed all suspended upstreams, unsuspend client
+ (and disable the write callback)
+- upstream close/error:
+ - look up pending ops, try to write to clients, mark clients suspended that
+ have ops that need responses (another queue associated with client to speed
+ up?)
+ - schedule a new connection open
+- client close/error:
+ - same as unbind
+- client inactive (no pending ops and nothing happened in x seconds)
+ - might just send notice of disconnection and close
+- op timeout handling:
+ - mark for abandon
+ - send abandon
+ - send timeLimitExceeded/adminLimitExceeded to client
+
+Picking an upstream:
+- while there is a level available:
+ - pick a random ordering of upstreams based on weights
+ - while there is an upstream in the level:
+ - check number of ops in-flight (this is where we lock the upstream map)
+ - find the least busy connection (and check if a new connection should be
+ opened)
+ - try to lock for socket write, if available (no BER queued) we have our
+ upstream
+
+PDU processing:
+- request (have an upstream selected):
+ - get new msgid from upstream
+ - create an Op structure (actually, with the need for freelist lock, we can
+ make it a cache for freed operation structures, avoiding some malloc
+ traffic, to reset, we need slap_sl_mem_create( ,,, 1 ))
+ - check proxyauthz is not present? or just let upstream reject it if there are
+ two?
+ - add own controls at the end:
+ - construct proxyauthz from authzid
+ - construct session tracking from remote IP, own name, authzid
+ - send over
+ - insert Op into client and upstream maps
+- response/intermediate/entry:
+ - look up Op in upstream's map
+ - write old msgid, rest of the response can go unchanged
+ - if a response, remove Op from all maps (client and upstream)
+
+Managing upstreams:
+- async connect up to min_connections (is there a point in having a connection
+ count range if we can't use it when needed since all of the below is async?)
+- when connected, set up TLS (if requested)
+- when done, send a bind
+- go for the bind interaction
+- when done, add it to the upstream's connection list
+- (if a connection is suspended or connections are over 75 % op limit, schedule
+ creating a new connection setup unless connection limit has been hit)
+
+Managing timeouts:
+- two options:
+ - maintain a separate locked priority queue to give a perfect ordering to when
+ each operation is to time out, would need to maintain yet another place
+ where operations can be found.
+ - the locking protocol for disposing of the operation would need to be
+ adjusted and might become even more complicated, might do the alternative
+ initially and then attempt this if it helps performance
+ - just do a sweep over all clients (that mutex is less contended) every so
+ often. With many in-flight operations might be a lot of wasted work.
+ - we still need to sweep over all clients to check if they should be killed
+ anyway
+
+Dispatcher thread (2^n of them, fd x is handled by thread no x % (2^n)):
+- poll on all registered fds
+- remove each fd that's ready from the registered list and schedule the work
+- work threads can put their fd back in if they deem necessary (=not suspended)
+- this works as a poor man's edge-triggered polling, with enough workers, should
+ we do proper edge triggered I/O? What about non-Linux?
+
+Listener thread:
+- slapd has just one, which then reassigns the sockets to separate I/O
+ threads
+
+Threading:
+- if using slap_sl_malloc, how much perf do we gain? To allocate a context per
+ op, we should have a dedicated parent context so that when we free it, we can
+ use that exclusively. The parent context's parent would be the main thread's
+ context. This implies a lot of slap_sl_mem_setctx/slap_sl_mem_create( ,,, 0 )
+ and making sure an op does not allocate/free things from two threads at the
+ same time (might need an Op mutex after all? Not such a huge cost if we
+ routinely reuse Op structures)
+
+Locking policy:
+- read mutexes are unnecessary, we only have one thread receiving data from the
+ connection - the one started from the dispatcher
+- two reference counters of operation structures (an op is accessible from
+ client and upstream map, each counter is consistent when thread has a lock on
+ corresponding map), when decreasing the counter to zero, start freeing
+ procedure
+- place to mark disposal finished for each side, consistency enforced by holding
+ the freelist lock when reading/manipulating
+- when op is created, we already have a write lock on upstream socket and map,
+ start writing, insert to upstream map with upstream refcount 1, unlock, lock
+ client, insert (client refcount 0), unlock, lock upstream, decrement refcount
+ (triggers a test if we need to drop it now), unlock upstream, done
+- when upstream processes a PDU, locks its map, increments counter, (potentially
+ removes if it's a response), unlocks, locks client's map, write mutex (this
+ order?) and full client mutex (if a bind response)
+- when client side wants to work with a PDU (abandon, (un)bind), locks its map,
+ increase refcount, unlocks, locks upstream map, write mutex, sends or queues
+ abandon, unlocks write mutex, initiates freeing procedure from upstream side
+ (or if having to remember we've already increased client-side refcount, mark
+ for deletion, lose upstream lock, lock client, decref, either triggering
+ deletion from client or mark for it)
+- if we have operation lock, we can simplify a bit (no need for three-stage
+ locking above)
+
+Shutdown:
+- stop accept() thread(s) - potentially add a channel to hand these listening
+ sockets over for zero-downtime restart
+- if very gentle, mark connections as closing, start timeout and:
+ - when a new non-abandon PDU comes in from client - return LDAP_UNAVAILABLE
+ - when receiving a PDU from upstream, send over to client, if no ops pending,
+ send unsolicited response and close (RFC4511 suggests unsolicited response
+ is the last PDU coming from the upstream and libldap agrees, so we can't
+ send it for a socket we want to shut down more gracefully)
+- gentle (or very gentle timed out):
+ - set timeout
+ - mark all ops as abandoned
+ - send unbind to all upstreams
+ - send unsolicited to all clients
+- imminent (or gentle timed out):
+ - async close all connections?
+ - exit()
+
+RootDSE:
+- default option is not to care and if a control/exop has special restrictions,
+ it is the admin's job to flag it as such in the load-balancer's config
+- another is not to care about the search request but check each search entry
+ being passed back, check DN and if it's a rootDSE, filter the list of
+ controls/exops/sasl mechs (external!) that are supported
+- last one is to check all search requests for the DN/scope and synthesise the
+ response locally - probably not (would need to configure the complete list of
+ controls, exops, sasl mechs, naming contexts in the balancer)
+
+Potential red flags:
+- we suspend upstreams, if we ever suspend clients we need to be sure we can't
+ create dependency cycles
+ - is this an issue when only suspending the read side of each? Because even if
+ we stop reading from everything, we should eventually flush data to those we
+ can still talk to, as upstreams are flushed, we can start sending new
+ requests from live clients (those that are suspended are due to their own
+ inability to accept data)
+ - we might need to suspend a client if there is a reason to choose a
+ particular upstream (multi-request operation - bind, VC, PR, TXN, ...)
+ - a SASL bind, but that means there are no outstanding ops to receive
+ it holds that !suspended(client) \or !suspended(upstream), so they
+ cannot participate in a cycle
+ - VC - multiple binds at the same time - !!! more analysis needed
+ - PR - should only be able to have one per connection (that's a problem
+ for later, maybe even needs a dedicated upstream connection)
+ - TXN - ??? probably same situation as PR
+ - or if we have a queue for pending Bers on the server, we not need to suspend
+ clients, upstream is only chosen if the queue is free or there is a reason
+ to send it to that particular upstream (multi-stage bind/VC, PR, ...), but
+ that still makes it possible for a client to exhaust all our memory by
+ sending requests (VC or other ones bound to a slow upstream or by not
+ reading the responses at all)
diff --git a/doc/devel/template.c b/doc/devel/template.c
new file mode 100644
index 0000000..28e028d
--- /dev/null
+++ b/doc/devel/template.c
@@ -0,0 +1,26 @@
+/* template.c -- example OpenLDAP source file */
+/* $OpenLDAP$ */
+/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
+ *
+ * Copyright YEAR The OpenLDAP Foundation.
+ * Portions Copyright YEAR Secondary Rights Holder.
+ * Portions Copyright YEAR Another Rights Holder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+/* Additional (custom) notices (where necessary).
+ * Please consult Kurt Zeilenga <kurt@openldap.org> before adding
+ * additional notices.
+ */
+/* ACKNOWLEDGEMENTS:
+ * This work was initially developed by Jane Doe for inclusion in
+ * OpenLDAP Software. Additional significant contributors include:
+ * John Doe
+ */
diff --git a/doc/devel/todo b/doc/devel/todo
new file mode 100644
index 0000000..670e9cc
--- /dev/null
+++ b/doc/devel/todo
@@ -0,0 +1,67 @@
+OpenLDAP Software To Do List
+----------------------------
+
+This is a list of projects that need getting done. They are defined
+by scale of the effort as opposed to priority. Contribute to
+projects based upon your personal priorities.
+
+If you would like to work on any of these projects, please coordinate
+by posting to OpenLDAP-devel mailing list:
+ http://www.OpenLDAP.org/lists
+
+If you have a project you'd like added to the list, talk it up on
+Developer's list or just do it.
+
+Please read:
+ http://www.OpenLDAP.org/devel/programming.html
+ http://www.OpenLDAP.org/devel/contributing.html
+
+
+OpenLDAP 2.x Projects
+---------------------
+ SLAPD
+ Complete Unicode Support (ACLs, etc.)
+ client C API update
+ Implement per referral/continuation callback
+ clients (e.g. ldapsearch(1))
+ Implement referral chasing options w/ referral callback
+ Update manual pages
+
+
+Large projects
+--------------
+Implement character string localization
+Implement X.500 administrative models (e.g. subentries (RFC 3672), etc.)
+Implement LDAP sorted search results control (RFC 2891)
+
+
+Medium projects
+---------------
+Add syncrepl turn
+Implement DIT Structure Rules and Name Forms
+Implement LDAPprep
+Implement native support for simple SASL mechanisms
+ (e.g. EXTERNAL and PLAIN)
+Redesign slapd memory allocation fault handling
+Localize tools
+
+
+Small projects
+--------------
+Add DSML capabilities to command line tools
+Add LDIFv2 (XML) support to command line tools
+Implement authPassword (RFC 3112)
+Implement SASLprep (RFC 4013) for LDAP (draft-ietf-ldapbis-*)
+Implement additional matching rules (RFC 3698)
+Add dumpasn1 logging support
+Add tests to test suite
+Recode linked-list structs to use <ldap_queue.h> macros
+Convert utfconv.txt into man page(s).
+Update manual pages as needed.
+
+
+For additional TODO items, see:
+ https://bugs.openldap.org
+
+---
+$OpenLDAP$
diff --git a/doc/devel/toolargs b/doc/devel/toolargs
new file mode 100644
index 0000000..f0f8d9f
--- /dev/null
+++ b/doc/devel/toolargs
@@ -0,0 +1,31 @@
+Tools ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz
+slapacl D F U X b d f o uv
+slapadd F S bcd fg j l no q s uvw
+slapauth F M R U X d f o v
+slapcat F H abcd fg l no s v
+slapdn F N P d f o v
+slapindex F bcd fg no q t v
+slapmodify F S bcd fg j l no q s uvw
+slappasswd T c h s uv
+slapschema F H abcd fg l no s v
+slaptest F Q d f no uv
+
+* General flags:
+ -F config directory
+ -U authcID
+ -X authzID
+ -b suffix (slapacl: entryDN)
+ -c continue mode
+ -d debug level
+ -f config file
+ -g disable subordinate gluing
+ -l LDIF file
+ -n database number
+ -o options
+ -q "quick" mode
+ -s disable schema checking (slapcat: subtree, slappasswd: secret)
+ -u dryrun (slappasswd: RFC2307 userPassword)
+ -v verbose
+
+---
+$OpenLDAP$
diff --git a/doc/devel/utfconv.txt b/doc/devel/utfconv.txt
new file mode 100644
index 0000000..1adaab5
--- /dev/null
+++ b/doc/devel/utfconv.txt
@@ -0,0 +1,291 @@
+ Dec 5, 2000
+ Dave Steck
+ Novell, Inc.
+
+ UTF-8 Conversion Functions
+
+
+1. Strings in the LDAP C SDK should be encoded in UTF-8 format.
+ However, most platforms do not provide APIs for converting to
+ this format. If they do, they are platform-specific.
+
+ As a result, most applications (knowingly or not) use local strings
+ with LDAP functions. This works fine for 7-bit ASCII characters,
+ but will fail with 8-bit European characters, Asian characters, etc.
+
+ We propose adding the following platform-independent conversion functions
+ to the OpenLDAP SDK. There are 4 functions for converting between UTF-8
+ and wide characters, and 4 functions for converting between UTF-8 and
+ multibyte characters.
+
+ For multibyte to UTF-8 conversions, charset translation is necessary.
+ While a full charset translator is not practical or appropriate for the
+ LDAP SDK, we can pass the translator function in as an argument.
+ A NULL for this argument will use the ANSI C functions mbtowc, mbstowcs,
+ wctomb, and wcstombs.
+
+2. UTF-8 <--> Wide Character conversions
+
+The following new conversion routines will be added, following the pattern of
+the ANSI C conversion routines (mbtowc, mbstowcs, etc). These routines use
+the wchar_t type. wchar_t is 2 bytes on some systems and 4 bytes on others.
+However the advantage of using wchar_t is that all the standard wide character
+string functions may be used on these strings: wcslen, wcscpy, etc.
+
+ int ldap_x_utf8_to_wc - Convert a single UTF-8 encoded character to a wide character.
+ int ldap_x_utf8s_to_wcs - Convert a UTF-8 string to a wide character string.
+ int ldap_x_wc_to_utf8 - Convert a single wide character to a UTF-8 sequence.
+ int ldap_x_wcs_to_utf8s - Convert a wide character string to a UTF-8 string.
+
+
+2.1 ldap_x_utf8_to_wc - Convert a single UTF-8 encoded character to a wide character.
+
+int ldap_x_utf8_to_wc ( wchar_t *wchar, const char *utf8char )
+
+ wchar (OUT) Points to a wide character code to receive the
+ converted character.
+
+ utf8char (IN) Address of the UTF8 sequence of bytes.
+
+Return Value:
+ If successful, the function returns the length in
+ bytes of the UTF-8 input character.
+
+ If utf8char is NULL or points to an empty string, the
+ function returns 1 and a NULL is written to wchar.
+
+ If utf8char contains an invalid UTF-8 sequence -1 is returned.
+
+
+2.2 ldap_x_utf8s_to_wcs - Convert a UTF-8 string to a wide character string.
+
+int ldap_x_utf8s_to_wcs (wchar_t *wcstr, const char *utf8str, size_t count)
+
+ wcstr (OUT) Points to a wide char buffer to receive the
+ converted wide char string. The output string will be
+ null terminated if there is space for it in the
+ buffer.
+
+ utf8str (IN) Address of the null-terminated UTF-8 string to convert.
+
+ count (IN) The number of UTF-8 characters to convert, or
+ equivalently, the size of the output buffer in wide
+ characters.
+
+Return Value:
+ If successful, the function returns the number of wide
+ characters written to wcstr, excluding the null termination
+ character, if any.
+
+ If wcstr is NULL, the function returns the number of wide
+ characters required to contain the converted string,
+ excluding the null termination character.
+
+ If an invalid UTF-8 sequence is encountered, the
+ function returns -1.
+
+ If the return value equals count, there was not enough space to fit the
+ string and the null terminator in the buffer.
+
+
+2.3 ldap_x_wc_to_utf8 - Convert a single wide character to a UTF-8 sequence.
+
+int ldap_x_wc_to_utf8 ( char *utf8char, wchar_t wchar, count )
+
+ utf8char (OUT) Points to a byte array to receive the converted UTF-8
+ string.
+
+ wchar (IN) The wide character to convert.
+
+ count (IN) The maximum number of bytes to write to the output
+ buffer. Normally set this to LDAP_MAX_UTF8_LEN, which
+ is defined as 3 or 6 depending on the size of wchar_t.
+ A partial character will not be written.
+
+Return Value:
+ If successful, the function returns the length in bytes of
+ the converted UTF-8 output character.
+
+ If wchar is NULL, the function returns 1 and a NULL is
+ written to utf8char.
+
+ If wchar cannot be converted to a UTF-8 character, the
+ function returns -1.
+
+
+2.4 int ldap_x_wcs_to_utf8s - Convert a wide character string to a UTF-8 string.
+
+int ldap_x_wcs_to_utf8s (char *utf8str, const wchar_t *wcstr, size_t count)
+
+ utf8str (OUT) Points to a byte array to receive the converted
+ UTF-8 string. The output string will be null
+ terminated if there is space for it in the
+ buffer.
+
+
+ wcstr (IN) Address of the null-terminated wide char string to convert.
+
+ count (IN) The size of the output buffer in bytes.
+
+Return Value:
+ If successful, the function returns the number of bytes
+ written to utf8str, excluding the null termination
+ character, if any.
+
+ If utf8str is NULL, the function returns the number of
+ bytes required to contain the converted string, excluding
+ the null termination character. The 'count' parameter is ignored.
+
+ If the function encounters a wide character that cannot
+ be mapped to a UTF-8 sequence, the function returns -1.
+
+ If the return value equals count, there was not enough space to fit
+ the string and the null terminator in the buffer.
+
+
+
+3. Multi-byte <--> UTF-8 Conversions
+
+These functions convert the string in a two-step process, from multibyte
+to Wide, then from Wide to UTF8, or vice versa. This conversion requires a
+charset translation routine, which is passed in as an argument.
+
+ ldap_x_mb_to_utf8 - Convert a multi-byte character to a UTF-8 character.
+ ldap_x_mbs_to_utf8s - Convert a multi-byte string to a UTF-8 string.
+ ldap_x_utf8_to_mb - Convert a UTF-8 character to a multi-byte character.
+ ldap_x_utf8s_to_mbs - Convert a UTF-8 string to a multi-byte string.
+
+3.1 ldap_x_mb_to_utf8 - Convert a multi-byte character to a UTF-8 character.
+
+int ldap_x_mb_to_utf8 ( char *utf8char, const char *mbchar, size_t mbsize, int (*f_mbtowc)(wchar_t *wchar, const char *mbchar, size_t count) )
+
+ utf8char (OUT) Points to a byte buffer to receive the converted
+ UTF-8 character. May be NULL. The output is not
+ null-terminated.
+
+ mbchar (IN) Address of a sequence of bytes forming a multibyte character.
+
+ mbsize (IN) The maximum number of bytes of the mbchar argument to
+ check. This should normally be MB_CUR_MAX.
+
+ f_mbtowc (IN) The function to use for converting a multibyte
+ character to a wide character. If NULL, the local
+ ANSI C routine mbtowc is used.
+
+Return Value:
+ If successful, the function returns the length in bytes of
+ the UTF-8 output character.
+
+ If utf8char is NULL, count is ignored and the function
+ returns the number of bytes that would be written to the
+ output char.
+
+ If count is zero, 0 is returned and nothing is written to
+ utf8char.
+
+ If mbchar is NULL or points to an empty string, the
+ function returns 1 and a null byte is written to utf8char.
+
+ If mbchar contains an invalid multi-byte character, -1 is returned.
+
+
+3.2 ldap_x_mbs_to_utf8s - Convert a multi-byte string to a UTF-8 string.
+
+int ldap_x_mbs_to_utf8s (char *utf8str, const char *mbstr, size_t count,
+ size_t (*f_mbstowcs)(wchar_t *wcstr, const char *mbstr, size_t count))
+
+utf8str (OUT) Points to a buffer to receive the converted UTF-8 string.
+ May be NULL.
+
+ mbchar (IN) Address of the null-terminated multi-byte input string.
+
+ count (IN) The size of the output buffer in bytes.
+
+ f_mbstowcs (IN) The function to use for converting a multibyte string
+ to a wide character string. If NULL, the local ANSI
+ C routine mbstowcs is used.
+
+Return Value:
+ If successful, the function returns the length in
+ bytes of the UTF-8 output string, excluding the null
+ terminator, if present.
+
+ If utf8str is NULL, count is ignored and the function
+ returns the number of bytes required for the output string,
+ excluding the NULL.
+
+ If count is zero, 0 is returned and nothing is written to utf8str.
+
+ If mbstr is NULL or points to an empty string, the
+ function returns 1 and a null byte is written to utf8str.
+
+ If mbstr contains an invalid multi-byte character, -1 is returned.
+
+ If the returned value is equal to count, the entire null-terminated
+ string would not fit in the output buffer.
+
+
+3.3 ldap_x_utf8_to_mb - Convert a UTF-8 character to a multi-byte character.
+
+int ldap_x_utf8_to_mb ( char *mbchar, const char *utf8char,
+ int (*f_wctomb)(char *mbchar, wchar_t wchar) )
+
+mbchar (OUT) Points to a byte buffer to receive the converted multi-byte
+ character. May be NULL.
+
+ utf8char (IN) Address of the UTF-8 character sequence.
+
+ f_wctomb (IN) The function to use for converting a wide character
+ to a multibyte character. If NULL, the local
+ ANSI C routine wctomb is used.
+
+
+Return Value:
+ If successful, the function returns the length in
+ bytes of the multi-byte output character.
+
+ If utf8char is NULL or points to an empty string, the
+ function returns 1 and a null byte is written to mbchar.
+
+ If utf8char contains an invalid UTF-8 sequence, -1 is returned.
+
+
+3.4 int ldap_x_utf8s_to_mbs - Convert a UTF-8 string to a multi-byte string.
+
+
+int ldap_x_utf8s_to_mbs ( char *mbstr, const char *utf8str, size_t count,
+ size_t (*f_wcstombs)(char *mbstr, const wchar_t *wcstr, size_t count) )
+
+ mbstr (OUT) Points to a byte buffer to receive the converted
+ multi-byte string. May be NULL.
+
+ utf8str (IN) Address of the null-terminated UTF-8 string to convert.
+
+ count (IN) The size of the output buffer in bytes.
+
+ f_wcstombs (IN) The function to use for converting a wide character
+ string to a multibyte string. If NULL, the local
+ ANSI C routine wcstombs is used.
+
+Return Value:
+ If successful, the function returns the number of bytes
+ written to mbstr, excluding the null termination
+ character, if any.
+
+ If mbstr is NULL, count is ignored and the function
+ returns the number of bytes required for the output string,
+ excluding the NULL.
+
+ If count is zero, 0 is returned and nothing is written to
+ mbstr.
+
+ If utf8str is NULL or points to an empty string, the
+ function returns 1 and a null byte is written to mbstr.
+
+ If an invalid UTF-8 character is encountered, the
+ function returns -1.
+
+The output string will be null terminated if there is space for it in
+the output buffer.
+
+
diff --git a/doc/devel/variadic_debug/03-libldap_Debug.cocci b/doc/devel/variadic_debug/03-libldap_Debug.cocci
new file mode 100644
index 0000000..8353e64
--- /dev/null
+++ b/doc/devel/variadic_debug/03-libldap_Debug.cocci
@@ -0,0 +1,70 @@
+using "equivalence.iso"
+
+@initialize:ocaml@
+@@
+// count the number of % characters in the format string
+let fmtn(fmt,n) =
+ List.length (Str.split_delim (Str.regexp_string "%") fmt) = n + 1
+
+# replace osip_debug/oslocal_debug with Debug() macros first
+@@
+expression E;
+expression list args;
+@@
+(
+-osip_debug
+|
+-oslocal_debug
+)
++Debug
+ (
+-E,
++LDAP_DEBUG_TRACE,
+ args );
+
+// replace Debug( ..., arg1, arg2, 0 ) with Debug2( ..., arg1, arg2 )
+@@
+char[] fmt : script:ocaml() { fmtn(fmt,2) };
+expression list[2] args;
+expression E;
+@@
+
+-Debug
++Debug2
+ ( E, _(fmt), args
+-, 0
+ );
+
+// replace Debug( ..., arg1, 0, 0 ) with Debug1()
+@@
+char[] fmt : script:ocaml() { fmtn(fmt,1) };
+expression list[1] args;
+expression E;
+@@
+
+-Debug
++Debug1
+ ( E, _(fmt), args
+-, 0, 0
+ );
+
+// Zero-argument Debug() -> Debug0()
+@@
+expression E, S;
+@@
+
+-Debug
++Debug0
+ ( E, S
+-, 0, 0, 0
+ );
+
+// everything else is a regular 3-argument debug macro, replace with Debug3()
+@@
+expression E, S;
+expression list[3] args;
+@@
+
+-Debug
++Debug3
+ ( E, S, args );
diff --git a/doc/devel/variadic_debug/04-variadic.cocci b/doc/devel/variadic_debug/04-variadic.cocci
new file mode 100644
index 0000000..bd5fbea
--- /dev/null
+++ b/doc/devel/variadic_debug/04-variadic.cocci
@@ -0,0 +1,165 @@
+@initialize:ocaml@
+@@
+// count the number of % characters in the format string
+let fmtn(fmt,n) =
+ List.length (Str.split_delim (Str.regexp_string "%") fmt) = n + 1
+
+@@
+identifier Logs =~ "Log[0-9]";
+@@
+-Logs
++Log
+
+@@
+@@
+-StatslogTest
++LogTest
+
+// Process two-argument Debug() macros with an extra zero
+@@
+char[] fmt : script:ocaml() { fmtn(fmt,2) };
+expression list[2] args;
+expression E;
+@@
+
+Debug( E, fmt, args
+-, 0
+ );
+
+@@
+char[] fmt : script:ocaml() { fmtn(fmt,2) };
+expression list[2] args;
+expression E;
+@@
+
+Debug( E, fmt, args
+-, NULL
+ );
+
+// Single argument Debug() macros with two extra zeroes
+@@
+char[] fmt : script:ocaml() { fmtn(fmt,1) };
+expression list[1] args;
+expression E;
+@@
+
+Debug( E, fmt, args
+-, 0, 0
+ );
+
+@@
+char[] fmt : script:ocaml() { fmtn(fmt,1) };
+expression list[1] args;
+expression E;
+@@
+
+Debug( E, fmt, args
+-, NULL, NULL
+ );
+
+// Debug() macros with no arguments just padded with zeroes
+@@
+expression E, S;
+@@
+
+Debug( E, S
+-, 0, 0, 0
+ );
+
+@@
+expression E, S;
+@@
+
+Debug( E, S
+-, NULL, NULL, NULL
+ );
+
+// Similar to above, just for Statslog
+@@
+char[] fmt : script:ocaml() { fmtn(fmt,5) };
+expression list[5] args;
+expression E;
+@@
+
+-Statslog
++Debug
+ ( E, fmt, args );
+
+@@
+char[] fmt : script:ocaml() { fmtn(fmt,4) };
+expression list[4] args;
+expression E;
+@@
+
+-Statslog
++Debug
+ ( E, fmt, args
+-, 0
+ );
+
+@@
+char[] fmt : script:ocaml() { fmtn(fmt,3) };
+expression list[3] args;
+expression E;
+@@
+
+-Statslog
++Debug
+ ( E, fmt, args
+-, 0, 0
+ );
+
+@@
+char[] fmt : script:ocaml() { fmtn(fmt,2) };
+expression list[2] args;
+expression E;
+@@
+
+-Statslog
++Debug
+ ( E, fmt, args
+-, 0, 0, 0
+ );
+
+@@
+char[] fmt : script:ocaml() { fmtn(fmt,1) };
+expression list[1] args;
+expression E;
+@@
+
+-Statslog
++Debug
+ ( E, fmt, args
+-, 0, 0, 0, 0
+ );
+
+@@
+expression E, S;
+@@
+
+-Statslog
++Debug
+ ( E, S
+-, 0, 0, 0, 0, 0
+ );
+
+// And StatslogEtime
+@@
+char[] fmt : script:ocaml() { fmtn(fmt,4) };
+expression list[4] args;
+expression E;
+@@
+
+StatslogEtime( E, fmt, args
+-, 0
+ );
+
+@@
+identifier Stats =~ "^Statslog";
+@@
+(
+ StatslogEtime
+|
+-Stats
++Debug
+)
diff --git a/doc/devel/variadic_debug/07-shortcut.cocci b/doc/devel/variadic_debug/07-shortcut.cocci
new file mode 100644
index 0000000..99b3b55
--- /dev/null
+++ b/doc/devel/variadic_debug/07-shortcut.cocci
@@ -0,0 +1,216 @@
+// Splice string `s` into the format string `fmtstring` replacing the
+// %-parameter at position `pos`
+@initialize:python@
+@@
+
+# regex from https://stackoverflow.com/questions/30011379/how-can-i-parse-a-c-format-string-in-python
+import re
+fmtstring = '''\
+( # start of capture group 1
+% # literal "%"
+(?: # first option
+(?:[-+0 #]{0,5}) # optional flags
+(?:\d+|\*)? # width
+(?:\.(?:\d+|\*))? # precision
+(?:h|l|ll|w|I|I32|I64)? # size
+[cCdiouxXeEfgGaAnpsSZ] # type
+) | # OR
+%%) # literal "%%"
+'''
+
+regex = re.compile(fmtstring, re.X)
+
+def parse_format(f):
+ return tuple((m.span(), m.group()) for m in
+ regex.finditer(f))
+
+def insert_at_pos(fmt, s, pos):
+ formats = parse_format(fmt)
+ span, format = formats[pos]
+ acc = fmt[:span[0]]
+ if s.startswith('"'):
+ acc += s[1:]
+ else:
+ acc += '" '
+ acc += s
+ if acc.endswith('"'):
+ acc = acc[:-1] + fmt[span[1]:]
+ else:
+ acc += ' "'
+ acc += fmt[span[1]:]
+ return acc
+
+// rest of the file implements the same as 09-merge.cocci
+// The main difference is that we only match on snprintf and Debug that are
+// directly adjacent, not based on control flow information which trips
+// coccinelle's model-checker
+@shortcut@
+identifier buf;
+expression E, L;
+expression list args_before, args, args_after;
+expression format1, format2;
+position p1, p2;
+@@
+
+snprintf@p1( buf, E, format1, args );
+Debug@p2( L, format2, args_before, buf, args_after );
+
+// use insert_at_pos above to construct the new format-string
+@script:python shortcut_process@
+format1 << shortcut.format1;
+format2 << shortcut.format2;
+args_before << shortcut.args_before;
+merged;
+@@
+
+pos = len(args_before.elements)
+coccinelle.merged = insert_at_pos(format2, format1, pos)
+
+@shortcut_replace@
+position shortcut.p1, shortcut.p2;
+identifier shortcut_process.merged;
+
+identifier buf;
+expression E, L;
+expression list args_before, args, args_after;
+expression format1, format2;
+@@
+
+-snprintf@p1( buf, E, format1, args );
+-Debug@p2( L, format2, args_before, buf, args_after );
++Debug( L, merged, args_before, args, args_after );
+
+@shortcut_locked@
+identifier buf;
+expression E, L, lock;
+expression list args_before, args, args_after;
+expression format1, format2;
+position p1, p2;
+@@
+
+ldap_pvt_thread_mutex_lock(lock);
+snprintf@p1( buf, E, format1, args );
+ldap_pvt_thread_mutex_unlock(lock);
+Debug@p2( L, format2, args_before, buf, args_after );
+
+// use insert_at_pos above to construct the new format-string
+@script:python shortcut_locked_process@
+format1 << shortcut_locked.format1;
+format2 << shortcut_locked.format2;
+args_before << shortcut_locked.args_before;
+merged;
+@@
+
+pos = len(args_before.elements)
+coccinelle.merged = insert_at_pos(format2, format1, pos)
+
+@shortcut_locked_replace@
+position shortcut_locked.p1, shortcut_locked.p2;
+identifier shortcut_locked_process.merged;
+
+identifier buf;
+expression E, L, lock;
+expression list args_before, args, args_after;
+expression format1, format2;
+@@
+
+ldap_pvt_thread_mutex_lock(lock);
+-snprintf@p1( buf, E, format1, args );
++Debug( L, merged, args_before, args, args_after );
+ldap_pvt_thread_mutex_unlock(lock);
+-Debug@p2( L, format2, args_before, buf, args_after );
+
+// so long as we don't reference 'buf' afterwards, no need to keep it defined.
+// A lot of pattern-matching is spelled out explicitly to work around the fact
+// that the state space doesn't get compressed otherwise.
+@@
+type T;
+identifier buf, id;
+expression E, lock;
+initializer I;
+@@
+{
+-\( T buf = I; \| T buf; \)
+(
+ ldap_pvt_thread_mutex_lock(lock);
+|
+)
+(
+ Debug( ... );
+&
+ ... when != buf
+)
+(
+ ldap_pvt_thread_mutex_unlock(lock);
+|
+)
+(
+|
+ continue;
+|
+ break;
+|
+ goto id;
+|
+ \(
+ return E;
+ \&
+ ... when != buf
+ \)
+)
+}
+
+// the rest identifies and removes a (newly-)redundant LogTest check
+@if_guard@
+position p;
+statement s;
+@@
+
+(
+ if ( ... ) {@p
+ Debug( ... );
+ } else s
+|
+ if ( ... ) {@p
+ Debug( ... );
+ }
+)
+
+@else_guard@
+position p;
+statement s;
+@@
+
+if ( ... ) s
+else {@p
+ Debug( ... );
+}
+
+@loop_guard@
+position p;
+@@
+
+(
+ while ( ... ) {@p
+ Debug( ... );
+ }
+|
+ for ( ...;...;... ) {@p
+ Debug( ... );
+ }
+)
+
+@@
+position p != { if_guard.p , else_guard.p, loop_guard.p };
+@@
+-{@p
+ Debug( ... );
+-}
+
+@useless_if@
+expression L;
+@@
+
+-if ( LogTest( L ) ) {
+ Debug( L, ... );
+-}
diff --git a/doc/devel/variadic_debug/09-merge.cocci b/doc/devel/variadic_debug/09-merge.cocci
new file mode 100644
index 0000000..4b0c1b2
--- /dev/null
+++ b/doc/devel/variadic_debug/09-merge.cocci
@@ -0,0 +1,147 @@
+// Note that this file has not actually been used in the end, since
+// 07-shortcut.cocci covers everything we needed in the project, but being
+// simpler, it makes the intent of 07-shortcut.cocci clearer
+
+
+// Splice string `s` into the format string `fmtstring` replacing the
+// %-parameter at position `pos`
+@initialize:python@
+@@
+
+#regex from https://stackoverflow.com/questions/30011379/how-can-i-parse-a-c-format-string-in-python
+import re
+fmtstring = '''\
+( # start of capture group 1
+% # literal "%"
+(?: # first option
+(?:[-+0 #]{0,5}) # optional flags
+(?:\d+|\*)? # width
+(?:\.(?:\d+|\*))? # precision
+(?:h|l|ll|w|I|I32|I64)? # size
+[cCdiouxXeEfgGaAnpsSZ] # type
+) | # OR
+%%) # literal "%%"
+'''
+
+regex = re.compile(fmtstring, re.X)
+
+def parse_format(f):
+ return tuple((m.span(), m.group()) for m in
+ regex.finditer(f))
+
+def insert_at_pos(fmt, s, pos):
+ formats = parse_format(fmt)
+ span, format = formats[pos]
+ acc = fmt[:span[0]]
+ if s.startswith('"'):
+ acc += s[1:]
+ else:
+ acc += '" '
+ acc += s
+ if acc.endswith('"'):
+ acc = acc[:-1] + fmt[span[1]:]
+ else:
+ acc += ' "'
+ acc += fmt[span[1]:]
+ return acc
+
+// Identify the redundant snprintfs (within a locked region)
+@a exists@
+expression lock, E, L;
+expression list args_before, args, args_after;
+identifier buf;
+expression format1, format2;
+type T;
+position p1, p2;
+@@
+
+{
+...
+T buf;
+...
+ldap_pvt_thread_mutex_lock(lock);
+...
+snprintf@p1( buf, E, format1, args );
+...
+ldap_pvt_thread_mutex_unlock(lock);
+...
+Debug@p2( L, format2, args_before, buf, args_after );
+...
+}
+
+// Merge the format strings with insert_at_pos above
+@script:python a_process@
+format1 << a.format1;
+format2 << a.format2;
+args_before << a.args_before;
+merged;
+@@
+
+pos = len(args_before.elements)
+coccinelle.merged = insert_at_pos(format2, format1, pos)
+
+// And merge the two together, replacing the extra buffer that's not used anymore
+@a_replace@
+position a.p1, a.p2;
+identifier a_process.merged;
+
+expression lock, E, L;
+expression list args_before, args, args_after;
+identifier buf;
+expression format1, format2;
+type T;
+@@
+
+{
+...
+-T buf;
+...
+ldap_pvt_thread_mutex_lock(lock);
+...
+-snprintf@p1( buf, E, format1, args );
++Debug( L, merged, args_before, args, args_after );
+...
+ldap_pvt_thread_mutex_unlock(lock);
+...
+-Debug@p2( L, format2, args_before, buf, args_after );
+...
+}
+
+// Once again (same as the 'a' series above, but those that remain to be sorted
+// now don't need to stay within a locked region
+@b exists@
+expression E, L;
+expression list args_before, args, args_after;
+identifier buf;
+expression format1, format2;
+position p1, p2;
+@@
+
+snprintf@p1( buf, E, format1, args );
+...
+Debug@p2( L, format2, args_before, buf, args_after );
+
+@script:python b_process@
+format1 << b.format1;
+format2 << b.format2;
+args_before << b.args_before;
+merged;
+@@
+
+pos = len(args_before.elements)
+coccinelle.merged = insert_at_pos(format2, format1, pos)
+
+@b_replace@
+position b.p1, b.p2;
+identifier b_process.merged;
+
+expression E, L;
+expression list args_before, args, args_after;
+identifier buf;
+expression format1, format2;
+@@
+
+-snprintf@p1( buf, E, format1, args );
++Debug( L, merged, args_before, args, args_after );
+...
+-Debug@p2( L, format2, args_before, buf, args_after );
diff --git a/doc/devel/variadic_debug/README b/doc/devel/variadic_debug/README
new file mode 100644
index 0000000..3ccbea2
--- /dev/null
+++ b/doc/devel/variadic_debug/README
@@ -0,0 +1,39 @@
+Most of the project now depends on the compiler supporting C99 variadic
+macros. This is used in the Debug() macro everywhere except libldap and
+its dependencies.
+
+From now on, any time Debug( level, fmt, args... ) is used, you can and
+should provide the appropriate number of arguments. The coccinelle
+patches in this directory implement the transformations used to bring
+the project in line with this.
+
+As we still aim to support libldap on platforms that only provide C89,
+Debug0/1/2/3 macros are used instead.
+
+If you need to adapt your own fork, see ITS#8731, the rest of this
+README and scripts in this directory on what you'll need to achieve
+this.
+
+Coccinelle as of git hash e65a7bdc04ac9122acdae2353422c5736b7998ba from
+https://github.com/coccinelle/coccinelle has been used to run the
+transformations performed. One notable limitation at the time of writing
+is that multi-part (format) strings are always merged onto the same line.
+
+Some sources cannot be processed, nssov overlay being a prime example,
+being wrapped in non-trivial macros.
+
+The following semantic patches are involved:
+- 03-libldap_Debug.cocci: converts the libraries to use the Debug[0123]
+ macros as appropriate
+- 04-variadic.cocci: converts the rest of the project to use the Debug
+ macro with the right number of arguments (as opposed to padding with
+ zeroes)
+- 09-merge.cocci will merge an 'snprintf(s, len, "fmt", args...);
+ Debug(level, "... %s ...", ..., s, ...);' sequence together
+- 07-shortcut.cocci is actually used to apply the above since
+ coccinelle's model-checker seems to struggle with state space
+ explosion in some of the very long and complex functions we have -
+ 09-merge.cocci doesn't finish in any reasonable time
+
+The equivalence.iso and macros.h files aid coccinelle to parse our
+sources correctly and simplify the semantic patches.
diff --git a/doc/devel/variadic_debug/equivalence.iso b/doc/devel/variadic_debug/equivalence.iso
new file mode 100644
index 0000000..07372fb
--- /dev/null
+++ b/doc/devel/variadic_debug/equivalence.iso
@@ -0,0 +1,12 @@
+Expression
+@ NULL @
+@@
+
+NULL <=> 0
+
+Expression
+@ underscore_func @
+expression E;
+@@
+
+_(E) => E
diff --git a/doc/devel/variadic_debug/macros.h b/doc/devel/variadic_debug/macros.h
new file mode 100644
index 0000000..265c549
--- /dev/null
+++ b/doc/devel/variadic_debug/macros.h
@@ -0,0 +1,23 @@
+#define LDAP_PF_LOCAL_SENDMSG_ARG(x)
+
+#define LDAP_P(x) x
+#define LDAP_F(x) extern x
+#define LDAP_V(x) extern x
+
+#define LDAP_GCCATTR(x)
+#define LDAP_XSTRING(x) ""
+#define LDAP_CONCAT(x,y) x
+
+#define LDAP_CONST const
+#define LDAP_BEGIN_DECL
+#define LDAP_END_DECL
+
+#define SLAP_EVENT_DECL
+#define SLAP_EVENT_FNAME
+
+/* contrib/slapd-modules/smbk5pwd/smbk5pwd.c */
+#define HDB int*
+
+#define BACKSQL_ARBITRARY_KEY
+#define BACKSQL_IDNUMFMT "%llu"
+#define BACKSQL_IDFMT "%s"
diff --git a/doc/devel/variadic_debug/script.sh b/doc/devel/variadic_debug/script.sh
new file mode 100755
index 0000000..b9fd9f0
--- /dev/null
+++ b/doc/devel/variadic_debug/script.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+
+set -e
+
+PATCH_DIR=doc/devel/variadic_debug
+
+SPATCH=${SPATCH:-spatch}
+SPATCH_OPTS=( --macro-file-builtins "$PATCH_DIR/macros.h" )
+#SPATCH_OPTS+=( --timeout 300 )
+
+SED_TRANSFORMATIONS=()
+
+# split out multipart strings back to original form (one per line)
+SED_TRANSFORMATIONS+=( -e 's/^\(+\s*\)\(.*"\) \(".*\)"$/\1\2\n+\1\3/' )
+
+# re-add whitespace around parentheses
+SED_TRANSFORMATIONS+=( -e 's/^\(+.*Debug[0-3]\?(\)\s*/\1 /' )
+SED_TRANSFORMATIONS+=( -e 's/^\(+.*[^ ]\));$/\1 );/' )
+
+# strip trailing whitespace copied from source on affected lines
+SED_TRANSFORMATIONS+=( -e 's/^\(+.*\)\s\+$/\1/' )
+
+# fix whitespace errors in source we touch
+SED_TRANSFORMATIONS+=( -e 's/^\(+.*\) \t/\1\t\t/' )
+SED_TRANSFORMATIONS+=( -e 's/^\(+\t*\) \{1,3\}\t/\1\t/' )
+
+normalise() {
+ patch="$1"
+ shift
+
+ # iterate until we've reached fixpoint
+ while ! cmp "$patch" "${patch}.new" 2>/dev/null; do
+ if [ -e "${patch}.new" ]; then
+ mv -- "${patch}.new" "$patch"
+ fi
+ sed "${SED_TRANSFORMATIONS[@]}" -- "$patch" >"${patch}.new"
+ done
+ rediff "$patch" >"${patch}.new"
+ mv -- "${patch}.new" "$patch"
+}
+
+git add "$PATCH_DIR"
+git commit -m "ITS#8731 Add the documentation and scripts"
+
+git am "$PATCH_DIR/00-fixes.patch"
+git am "$PATCH_DIR/01-logging.patch"
+git am "$PATCH_DIR/02-manual.patch"
+
+$SPATCH "${SPATCH_OPTS[@]}" -sp_file "$PATCH_DIR/03-libldap_Debug.cocci" \
+ -dir libraries/libldap \
+ >"$PATCH_DIR/03-libldap_Debug.patch"
+normalise "$PATCH_DIR/03-libldap_Debug.patch"
+git apply --index --directory libraries/libldap "$PATCH_DIR/03-libldap_Debug.patch"
+git commit -m "ITS#8731 Apply $PATCH_DIR/03-libldap_Debug.cocci"
+
+$SPATCH "${SPATCH_OPTS[@]}" -sp_file "$PATCH_DIR/04-variadic.cocci" \
+ -dir . \
+ >"$PATCH_DIR/04-variadic.patch"
+normalise "$PATCH_DIR/04-variadic.patch"
+git apply --index "$PATCH_DIR/04-variadic.patch"
+git commit -m "ITS#8731 Apply $PATCH_DIR/04-variadic.cocci"
+
+git am "$PATCH_DIR/05-back-sql.patch"
+git am "$PATCH_DIR/06-nssov.patch"
+
+$SPATCH "${SPATCH_OPTS[@]}" -sp_file "$PATCH_DIR/07-shortcut.cocci" \
+ -dir . \
+ >"$PATCH_DIR/07-shortcut.patch"
+normalise "$PATCH_DIR/07-shortcut.patch"
+git apply --index "$PATCH_DIR/07-shortcut.patch"
+git commit -m "ITS#8731 Apply $PATCH_DIR/07-shortcut.cocci"
+
+git am "$PATCH_DIR/08-snprintf-manual.patch"