Adding upstream version 2:4.20.0+dfsg.upstream/2%4.20.0+dfsg

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

8 files changed, 2865 insertions, 0 deletions

diff --git a/ctdb/ib/README.txt b/ctdb/ib/README.txt
new file mode 100644
index 0000000..e238a1d
--- /dev/null
+++ b/ctdb/ib/README.txt
@@ -0,0 +1,10 @@
+Compilation
+===========
+
+For the configure script, please set the OFED include & library path by e.g.:
+
+export CFLAGS="-I/usr/local/ofed/include -L/usr/local/ofed/lib"
+
+After then:
+
+./configure --enable-infiniband
diff --git a/ctdb/ib/ibw_ctdb.c b/ctdb/ib/ibw_ctdb.c
new file mode 100644
index 0000000..38314c3
--- /dev/null
+++ b/ctdb/ib/ibw_ctdb.c
@@ -0,0 +1,185 @@
+/*
+ * Unix SMB/CIFS implementation.
+ * Join infiniband wrapper and ctdb.
+ *
+ * Copyright (C) Sven Oehme <oehmes@de.ibm.com> 2006
+ *
+ * Major code contributions by Peter Somogyi <psomogyi@gamax.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "replace.h"
+#include "system/network.h"
+
+#include <assert.h>
+#include <talloc.h>
+#include <tevent.h>
+
+#include "lib/util/time.h"
+#include "lib/util/debug.h"
+
+#include "ctdb_private.h"
+
+#include "common/common.h"
+#include "common/logging.h"
+
+#include "ibwrapper.h"
+#include "ibw_ctdb.h"
+
+int ctdb_ibw_get_address(struct ctdb_context *ctdb,
+	const char *address, struct in_addr *addr)
+{
+	if (inet_pton(AF_INET, address, addr) <= 0) {
+		struct hostent *he = gethostbyname(address);
+		if (he == NULL || he->h_length > sizeof(*addr)) {
+			ctdb_set_error(ctdb, "invalid network address '%s'\n", 
+				       address);
+			return -1;
+		}
+		memcpy(addr, he->h_addr, he->h_length);
+	}
+	return 0;
+}
+
+int ctdb_ibw_node_connect(struct ctdb_node *node)
+{
+	struct ctdb_ibw_node *cn = talloc_get_type(node->transport_data,
+						   struct ctdb_ibw_node);
+	int	rc;
+
+	assert(cn!=NULL);
+	assert(cn->conn!=NULL);
+
+	rc = ibw_connect(cn->conn, &node->address.ip, node);
+	if (rc) {
+		DEBUG(DEBUG_ERR, ("ctdb_ibw_node_connect/ibw_connect failed - retrying...\n"));
+		/* try again once a second */
+		tevent_add_timer(node->ctdb->ev, node,
+				 timeval_current_ofs(1, 0),
+				 ctdb_ibw_node_connect_event, node);
+	}
+
+	/* continues at ibw_ctdb.c/IBWC_CONNECTED in good case */
+	return 0;
+}
+
+void ctdb_ibw_node_connect_event(struct tevent_context *ev,
+				 struct tevent_timer *te,
+				 struct timeval t, void *private_data)
+{
+	struct ctdb_node *node = talloc_get_type(private_data, struct ctdb_node);
+
+	ctdb_ibw_node_connect(node);
+}
+
+int ctdb_ibw_connstate_handler(struct ibw_ctx *ctx, struct ibw_conn *conn)
+{
+	if (ctx!=NULL) {
+		/* ctx->state changed */
+		switch(ctx->state) {
+		case IBWS_INIT: /* ctx start - after ibw_init */
+			break;
+		case IBWS_READY: /* after ibw_bind & ibw_listen */
+			break;
+		case IBWS_CONNECT_REQUEST: /* after [IBWS_READY + incoming request] */
+				/* => [(ibw_accept)IBWS_READY | (ibw_disconnect)STOPPED | ERROR] */
+			if (ibw_accept(ctx, conn, NULL)) {
+				DEBUG(DEBUG_ERR, ("connstate_handler/ibw_accept failed\n"));
+				return -1;
+			} /* else continue in IBWC_CONNECTED */
+			break;
+		case IBWS_STOPPED: /* normal stop <= ibw_disconnect+(IBWS_READY | IBWS_CONNECT_REQUEST) */
+			/* TODO: have a CTDB upcall for which CTDB should wait in a (final) loop */
+			break;
+		case IBWS_ERROR: /* abnormal state; ibw_stop must be called after this */
+			break;
+		default:
+			assert(0);
+			break;
+		}
+	}
+
+	if (conn!=NULL) {
+		/* conn->state changed */
+		switch(conn->state) {
+		case IBWC_INIT: /* conn start - internal state */
+			break;
+		case IBWC_CONNECTED: { /* after ibw_accept or ibw_connect */
+			struct ctdb_node *node = talloc_get_type(conn->conn_userdata, struct ctdb_node);
+			if (node!=NULL) { /* after ibw_connect */
+				struct ctdb_ibw_node *cn = talloc_get_type(
+					node->transport_data,
+					struct ctdb_ibw_node);
+
+				node->ctdb->upcalls->node_connected(node);
+				ctdb_flush_cn_queue(cn);
+			} else { /* after ibw_accept */
+				/* NOP in CTDB case */
+			}
+		} break;
+		case IBWC_DISCONNECTED: { /* after ibw_disconnect */
+			struct ctdb_node *node = talloc_get_type(conn->conn_userdata, struct ctdb_node);
+			if (node!=NULL)
+				node->ctdb->upcalls->node_dead(node);
+			talloc_free(conn);
+			/* normal + intended disconnect => not reconnecting in this layer */
+		} break;
+		case IBWC_ERROR: {
+			struct ctdb_node *node = talloc_get_type(conn->conn_userdata, struct ctdb_node);
+			if (node!=NULL) {
+				struct ctdb_ibw_node *cn = talloc_get_type(
+					node->transport_data,
+					struct ctdb_ibw_node);
+				struct ibw_ctx *ictx = cn->conn->ctx;
+
+				DEBUG(DEBUG_DEBUG, ("IBWC_ERROR, reconnecting...\n"));
+				talloc_free(cn->conn); /* internal queue content is destroyed */
+				cn->conn = (void *)ibw_conn_new(ictx, node);
+				tevent_add_timer(node->ctdb->ev, node,
+						 timeval_current_ofs(1, 0),
+						 ctdb_ibw_node_connect_event, node);
+			}
+		} break;
+		default:
+			assert(0);
+			break;
+		}
+	}
+
+	return 0;
+}
+
+int ctdb_ibw_receive_handler(struct ibw_conn *conn, void *buf, int n)
+{
+	struct ctdb_context *ctdb = talloc_get_type(conn->ctx->ctx_userdata, struct ctdb_context);
+	void	*buf2; /* future TODO: a solution for removal of this */
+
+	assert(ctdb!=NULL);
+	assert(buf!=NULL);
+	assert(conn!=NULL);
+	assert(conn->state==IBWC_CONNECTED);
+
+	/* so far "buf" is an ib-registered memory area
+	 * and being reused for next receive
+	 * noticed that HL requires talloc-ed memory to be stolen */
+	buf2 = talloc_zero_size(conn, n);
+	CTDB_NO_MEMORY(ctdb, buf2);
+
+	memcpy(buf2, buf, n);
+
+	ctdb->upcalls->recv_pkt(ctdb, (uint8_t *)buf2, (uint32_t)n);
+
+	return 0;
+}
diff --git a/ctdb/ib/ibw_ctdb.h b/ctdb/ib/ibw_ctdb.h
new file mode 100644
index 0000000..57f659b
--- /dev/null
+++ b/ctdb/ib/ibw_ctdb.h
@@ -0,0 +1,51 @@
+/*
+ * Unix SMB/CIFS implementation.
+ * Join infiniband wrapper and ctdb.
+ *
+ * Copyright (C) Sven Oehme <oehmes@de.ibm.com> 2006
+ *
+ * Major code contributions by Peter Somogyi <psomogyi@gamax.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+struct ctdb_ibw_msg {
+	uint8_t *data;
+	uint32_t length;
+	struct ctdb_ibw_msg *prev;
+	struct ctdb_ibw_msg *next;
+};
+
+struct ctdb_ibw_node {
+	struct ibw_conn *conn;
+
+	struct ctdb_ibw_msg *queue;
+	struct ctdb_ibw_msg *queue_last;
+	int	qcnt;
+};
+
+int ctdb_ibw_get_address(struct ctdb_context *ctdb,
+	const char *address, struct in_addr *addr);
+
+int ctdb_ibw_connstate_handler(struct ibw_ctx *ctx, struct ibw_conn *conn);
+int ctdb_ibw_receive_handler(struct ibw_conn *conn, void *buf, int n);
+
+int ctdb_ibw_node_connect(struct ctdb_node *node);
+void ctdb_ibw_node_connect_event(struct tevent_context *ev,
+				 struct tevent_timer *te,
+				 struct timeval t, void *private_data);
+
+int ctdb_flush_cn_queue(struct ctdb_ibw_node *cn);
+
+int ctdb_ibw_init(struct ctdb_context *ctdb);
diff --git a/ctdb/ib/ibw_ctdb_init.c b/ctdb/ib/ibw_ctdb_init.c
new file mode 100644
index 0000000..f9d00c6
--- /dev/null
+++ b/ctdb/ib/ibw_ctdb_init.c
@@ -0,0 +1,255 @@
+/*
+ * Unix SMB/CIFS implementation.
+ * Join infiniband wrapper and ctdb.
+ *
+ * Copyright (C) Sven Oehme <oehmes@de.ibm.com> 2006
+ *
+ * Major code contributions by Peter Somogyi <psomogyi@gamax.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "replace.h"
+#include "system/network.h"
+
+#include <assert.h>
+#include <talloc.h>
+#include <tevent.h>
+
+#include "lib/util/dlinklist.h"
+#include "lib/util/debug.h"
+
+#include "ctdb_private.h"
+
+#include "common/common.h"
+#include "common/logging.h"
+
+#include "ibwrapper.h"
+#include "ibw_ctdb.h"
+
+static int ctdb_ibw_listen(struct ctdb_context *ctdb, int backlog)
+{
+	struct ibw_ctx *ictx = talloc_get_type(ctdb->transport_data,
+					       struct ibw_ctx);
+
+	assert(ictx!=NULL);
+
+	if (ibw_bind(ictx, &ctdb->address->ip)) {
+		DEBUG(DEBUG_CRIT, ("ctdb_ibw_listen: ibw_bind failed\n"));
+		return -1;
+	}
+
+	if (ibw_listen(ictx, backlog)) {
+		DEBUG(DEBUG_CRIT, ("ctdb_ibw_listen: ibw_listen failed\n"));
+		return -1;
+	}
+
+	return 0;
+}
+
+/*
+ * initialise ibw portion of a ctdb node 
+ */
+static int ctdb_ibw_add_node(struct ctdb_node *node)
+{
+	struct ibw_ctx *ictx = talloc_get_type(node->ctdb->transport_data,
+					       struct ibw_ctx);
+	struct ctdb_ibw_node *cn = talloc_zero(node, struct ctdb_ibw_node);
+
+	assert(cn!=NULL);
+	cn->conn = ibw_conn_new(ictx, node);
+	node->transport_data = (void *)cn;
+
+	return (cn->conn!=NULL ? 0 : -1);
+}
+
+/*
+ * initialise infiniband
+ */
+static int ctdb_ibw_initialise(struct ctdb_context *ctdb)
+{
+	int i, ret;
+
+	ret = ctdb_ibw_init(ctdb);
+	if (ret != 0) {
+		return ret;
+	}
+
+	for (i=0; i<ctdb->num_nodes; i++) {
+		if (ctdb_ibw_add_node(ctdb->nodes[i]) != 0) {
+			DEBUG(DEBUG_CRIT, ("methods->add_node failed at %d\n", i));
+			return -1;
+		}
+	}
+
+	/* listen on our own address */
+	if (ctdb_ibw_listen(ctdb, 10)) /* TODO: backlog as param */
+		return -1;
+
+	return 0;
+}
+
+
+/*
+ * Start infiniband
+ */
+static int ctdb_ibw_start(struct ctdb_context *ctdb)
+{
+	int i;
+
+	/* everything async here */
+	for (i=0;i<ctdb->num_nodes;i++) {
+		struct ctdb_node *node = ctdb->nodes[i];
+		if (!ctdb_same_address(ctdb->address, &node->address)) {
+			ctdb_ibw_node_connect(node);
+		}
+	}
+
+	return 0;
+}
+
+static int ctdb_ibw_send_pkt(struct ibw_conn *conn, uint8_t *data, uint32_t length)
+{
+	void	*buf, *key;
+
+	if (ibw_alloc_send_buf(conn, &buf, &key, length)) {
+		DEBUG(DEBUG_ERR, ("queue_pkt/ibw_alloc_send_buf failed\n"));
+		return -1;
+	}
+
+	memcpy(buf, data, length);
+	return ibw_send(conn, buf, key, length);
+}
+
+int ctdb_flush_cn_queue(struct ctdb_ibw_node *cn)
+{
+	struct ctdb_ibw_msg *p;
+	int	rc = 0;
+
+	while(cn->queue) {
+		p = cn->queue;
+		rc = ctdb_ibw_send_pkt(cn->conn, p->data, p->length);
+		if (rc)
+			return -1; /* will be retried later when conn is up */
+
+		DLIST_REMOVE(cn->queue, p);
+		cn->qcnt--;
+		talloc_free(p); /* it will talloc_free p->data as well */
+	}
+	assert(cn->qcnt==0);
+	/* cn->queue_last = NULL is not needed - see DLIST_ADD_AFTER */
+
+	return rc;
+}
+
+static int ctdb_ibw_queue_pkt(struct ctdb_node *node, uint8_t *data, uint32_t length)
+{
+	struct ctdb_ibw_node *cn = talloc_get_type(node->transport_data,
+						   struct ctdb_ibw_node);
+	int	rc;
+
+	assert(length>=sizeof(uint32_t));
+	assert(cn!=NULL);
+
+	if (cn->conn==NULL) {
+		DEBUG(DEBUG_ERR, ("ctdb_ibw_queue_pkt: conn is NULL\n"));
+		return -1;
+	}
+
+	if (cn->conn->state==IBWC_CONNECTED) {
+		rc = ctdb_ibw_send_pkt(cn->conn, data, length);
+	} else {
+		struct ctdb_ibw_msg *p = talloc_zero(cn, struct ctdb_ibw_msg);
+		CTDB_NO_MEMORY(node->ctdb, p);
+
+		p->data = talloc_memdup(p, data, length);
+		CTDB_NO_MEMORY(node->ctdb, p->data);
+
+		p->length = length;
+
+		DLIST_ADD_AFTER(cn->queue, p, cn->queue_last);
+		cn->queue_last = p;
+		cn->qcnt++;
+
+		rc = 0;
+	}
+
+	return rc;
+}
+
+static void ctdb_ibw_restart(struct ctdb_node *node)
+{
+	/* TODO: implement this method for IB */
+	DEBUG(DEBUG_ALERT,("WARNING: method restart is not yet implemented for IB\n"));
+}
+
+/*
+ * transport packet allocator - allows transport to control memory for packets
+ */
+static void *ctdb_ibw_allocate_pkt(TALLOC_CTX *mem_ctx, size_t size)
+{
+	/* TODO: use ibw_alloc_send_buf instead... */
+	return talloc_size(mem_ctx, size);
+}
+
+#ifdef __NOTDEF__
+
+static int ctdb_ibw_stop(struct ctdb_context *cctx)
+{
+	struct ibw_ctx *ictx = talloc_get_type(cctx->private_data, struct ibw_ctx);
+
+	assert(ictx!=NULL);
+	return ibw_stop(ictx);
+}
+
+#endif /* __NOTDEF__ */
+
+static const struct ctdb_methods ctdb_ibw_methods = {
+	.initialise= ctdb_ibw_initialise,
+	.start     = ctdb_ibw_start,
+	.queue_pkt = ctdb_ibw_queue_pkt,
+	.add_node = ctdb_ibw_add_node,
+	.allocate_pkt = ctdb_ibw_allocate_pkt,
+	.restart      = ctdb_ibw_restart,
+
+//	.stop = ctdb_ibw_stop
+};
+
+/*
+ * initialise ibw portion of ctdb 
+ */
+int ctdb_ibw_init(struct ctdb_context *ctdb)
+{
+	struct ibw_ctx *ictx;
+
+	DEBUG(DEBUG_DEBUG, ("ctdb_ibw_init invoked...\n"));
+	ictx = ibw_init(
+		NULL, //struct ibw_initattr *attr, /* TODO */
+		0, //int nattr, /* TODO */
+		ctdb,
+		ctdb_ibw_connstate_handler,
+		ctdb_ibw_receive_handler,
+		ctdb->ev);
+
+	if (ictx==NULL) {
+		DEBUG(DEBUG_CRIT, ("ctdb_ibw_init: ibw_init failed\n"));
+		return -1;
+	}
+
+	ctdb->methods = &ctdb_ibw_methods;
+	ctdb->transport_data = ictx;
+	
+	DEBUG(DEBUG_DEBUG, ("ctdb_ibw_init succeeded.\n"));
+	return 0;
+}
diff --git a/ctdb/ib/ibwrapper.c b/ctdb/ib/ibwrapper.c
new file mode 100644
index 0000000..cf4efa5
--- /dev/null
+++ b/ctdb/ib/ibwrapper.c
@@ -0,0 +1,1361 @@
+/*
+ * Unix SMB/CIFS implementation.
+ * Wrap Infiniband calls.
+ *
+ * Copyright (C) Sven Oehme <oehmes@de.ibm.com> 2006
+ *
+ * Major code contributions by Peter Somogyi <psomogyi@gamax.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "replace.h"
+#include "system/network.h"
+
+#include <assert.h>
+#include <talloc.h>
+#include <tevent.h>
+
+#include "lib/util/dlinklist.h"
+#include "lib/util/debug.h"
+
+#include "common/logging.h"
+
+#include <rdma/rdma_cma_abi.h>
+#include <rdma/rdma_cma.h>
+
+#include "ibwrapper.h"
+#include "ibwrapper_internal.h"
+
+#define IBW_LASTERR_BUFSIZE 512
+static char ibw_lasterr[IBW_LASTERR_BUFSIZE];
+
+#define IBW_MAX_SEND_WR 256
+#define IBW_MAX_RECV_WR 1024
+#define IBW_RECV_BUFSIZE 256
+#define IBW_RECV_THRESHOLD (1 * 1024 * 1024)
+
+static void ibw_event_handler_verbs(struct tevent_context *ev,
+	struct tevent_fd *fde, uint16_t flags, void *private_data);
+static int ibw_fill_cq(struct ibw_conn *conn);
+static int ibw_wc_recv(struct ibw_conn *conn, struct ibv_wc *wc);
+static int ibw_wc_send(struct ibw_conn *conn, struct ibv_wc *wc);
+static int ibw_send_packet(struct ibw_conn *conn, void *buf, struct ibw_wr *p, uint32_t len);
+
+static void *ibw_alloc_mr(struct ibw_ctx_priv *pctx, struct ibw_conn_priv *pconn,
+	uint32_t n, struct ibv_mr **ppmr)
+{
+	void *buf;
+
+	DEBUG(DEBUG_DEBUG, ("ibw_alloc_mr(cmid=%p, n=%u)\n", pconn->cm_id, n));
+	buf = memalign(pctx->pagesize, n);
+	if (!buf) {
+		sprintf(ibw_lasterr, "couldn't allocate memory\n");
+		return NULL;
+	}
+
+	*ppmr = ibv_reg_mr(pconn->pd, buf, n, IBV_ACCESS_LOCAL_WRITE);
+	if (!*ppmr) {
+		sprintf(ibw_lasterr, "couldn't allocate mr\n");
+		free(buf);
+		return NULL;
+	}
+
+	return buf;
+}
+
+static void ibw_free_mr(char **ppbuf, struct ibv_mr **ppmr)
+{
+	DEBUG(DEBUG_DEBUG, ("ibw_free_mr(%p %p)\n", *ppbuf, *ppmr));
+	if (*ppmr!=NULL) {
+		ibv_dereg_mr(*ppmr);
+		*ppmr = NULL;
+	}
+	if (*ppbuf) {
+		free(*ppbuf);
+		*ppbuf = NULL;
+	}
+}
+
+static int ibw_init_memory(struct ibw_conn *conn)
+{
+	struct ibw_ctx_priv *pctx = talloc_get_type(conn->ctx->internal, struct ibw_ctx_priv);
+	struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
+	struct ibw_opts *opts = &pctx->opts;
+	int	i;
+	struct ibw_wr	*p;
+
+	DEBUG(DEBUG_DEBUG, ("ibw_init_memory(cmid: %p)\n", pconn->cm_id));
+	pconn->buf_send = ibw_alloc_mr(pctx, pconn,
+		opts->max_send_wr * opts->recv_bufsize, &pconn->mr_send);
+	if (!pconn->buf_send) {
+		sprintf(ibw_lasterr, "couldn't allocate work send buf\n");
+		return -1;
+	}
+
+	pconn->buf_recv = ibw_alloc_mr(pctx, pconn,
+		opts->max_recv_wr * opts->recv_bufsize, &pconn->mr_recv);
+	if (!pconn->buf_recv) {
+		sprintf(ibw_lasterr, "couldn't allocate work recv buf\n");
+		return -1;
+	}
+
+	pconn->wr_index = talloc_size(pconn, opts->max_send_wr * sizeof(struct ibw_wr *));
+	assert(pconn->wr_index!=NULL);
+
+	for(i=0; i<opts->max_send_wr; i++) {
+		p = pconn->wr_index[i] = talloc_zero(pconn, struct ibw_wr);
+		p->buf = pconn->buf_send + (i * opts->recv_bufsize);
+		p->wr_id = i;
+
+		DLIST_ADD(pconn->wr_list_avail, p);
+	}
+
+	return 0;
+}
+
+static int ibw_ctx_priv_destruct(struct ibw_ctx_priv *pctx)
+{
+	DEBUG(DEBUG_DEBUG, ("ibw_ctx_priv_destruct(%p)\n", pctx));
+
+	/*
+	 * tevent_fd must be removed before the fd is closed
+	 */
+	TALLOC_FREE(pctx->cm_channel_event);
+
+	/* destroy cm */
+	if (pctx->cm_channel) {
+		rdma_destroy_event_channel(pctx->cm_channel);
+		pctx->cm_channel = NULL;
+	}
+	if (pctx->cm_id) {
+		rdma_destroy_id(pctx->cm_id);
+		pctx->cm_id = NULL;
+	}
+
+	return 0;
+}
+
+static int ibw_ctx_destruct(struct ibw_ctx *ctx)
+{
+	DEBUG(DEBUG_DEBUG, ("ibw_ctx_destruct(%p)\n", ctx));
+	return 0;
+}
+
+static int ibw_conn_priv_destruct(struct ibw_conn_priv *pconn)
+{
+	DEBUG(DEBUG_DEBUG, ("ibw_conn_priv_destruct(%p, cmid: %p)\n",
+		pconn, pconn->cm_id));
+
+	/* pconn->wr_index is freed by talloc */
+	/* pconn->wr_index[i] are freed by talloc */
+
+	/*
+	 * tevent_fd must be removed before the fd is closed
+	 */
+	TALLOC_FREE(pconn->verbs_channel_event);
+
+	/* destroy verbs */
+	if (pconn->cm_id!=NULL && pconn->cm_id->qp!=NULL) {
+		rdma_destroy_qp(pconn->cm_id);
+		pconn->cm_id->qp = NULL;
+	}
+
+	if (pconn->cq!=NULL) {
+		ibv_destroy_cq(pconn->cq);
+		pconn->cq = NULL;
+	}
+
+	if (pconn->verbs_channel!=NULL) {
+		ibv_destroy_comp_channel(pconn->verbs_channel);
+		pconn->verbs_channel = NULL;
+	}
+
+	/* free memory regions */
+	ibw_free_mr(&pconn->buf_send, &pconn->mr_send);
+	ibw_free_mr(&pconn->buf_recv, &pconn->mr_recv);
+
+	if (pconn->pd) {
+		ibv_dealloc_pd(pconn->pd);
+		pconn->pd = NULL;
+		DEBUG(DEBUG_DEBUG, ("pconn=%p pd deallocated\n", pconn));
+	}
+
+	if (pconn->cm_id) {
+		rdma_destroy_id(pconn->cm_id);
+		pconn->cm_id = NULL;
+		DEBUG(DEBUG_DEBUG, ("pconn=%p cm_id destroyed\n", pconn));
+	}
+
+	return 0;
+}
+
+static int ibw_wr_destruct(struct ibw_wr *wr)
+{
+	if (wr->buf_large!=NULL)
+		ibw_free_mr(&wr->buf_large, &wr->mr_large);
+	return 0;
+}
+
+static int ibw_conn_destruct(struct ibw_conn *conn)
+{
+	DEBUG(DEBUG_DEBUG, ("ibw_conn_destruct(%p)\n", conn));
+	
+	/* important here: ctx is a talloc _parent_ */
+	DLIST_REMOVE(conn->ctx->conn_list, conn);
+	return 0;
+}
+
+struct ibw_conn *ibw_conn_new(struct ibw_ctx *ctx, TALLOC_CTX *mem_ctx)
+{
+	struct ibw_conn *conn;
+	struct ibw_conn_priv *pconn;
+
+	assert(ctx!=NULL);
+
+	conn = talloc_zero(mem_ctx, struct ibw_conn);
+	assert(conn!=NULL);
+	talloc_set_destructor(conn, ibw_conn_destruct);
+
+	pconn = talloc_zero(conn, struct ibw_conn_priv);
+	assert(pconn!=NULL);
+	talloc_set_destructor(pconn, ibw_conn_priv_destruct);
+
+	conn->ctx = ctx;
+	conn->internal = (void *)pconn;
+
+	DLIST_ADD(ctx->conn_list, conn);
+
+	return conn;
+}
+
+static int ibw_setup_cq_qp(struct ibw_conn *conn)
+{
+	struct ibw_ctx_priv *pctx = talloc_get_type(conn->ctx->internal, struct ibw_ctx_priv);
+	struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
+	struct ibv_qp_init_attr init_attr;
+	struct ibv_qp_attr attr;
+	int rc;
+
+	DEBUG(DEBUG_DEBUG, ("ibw_setup_cq_qp(cmid: %p)\n", pconn->cm_id));
+
+	/* init verbs */
+	pconn->verbs_channel = ibv_create_comp_channel(pconn->cm_id->verbs);
+	if (!pconn->verbs_channel) {
+		sprintf(ibw_lasterr, "ibv_create_comp_channel failed %d\n", errno);
+		return -1;
+	}
+	DEBUG(DEBUG_DEBUG, ("created channel %p\n", pconn->verbs_channel));
+
+	pconn->verbs_channel_event = tevent_add_fd(pctx->ectx, NULL, /* not pconn or conn */
+		pconn->verbs_channel->fd, TEVENT_FD_READ, ibw_event_handler_verbs, conn);
+
+	pconn->pd = ibv_alloc_pd(pconn->cm_id->verbs);
+	if (!pconn->pd) {
+		sprintf(ibw_lasterr, "ibv_alloc_pd failed %d\n", errno);
+		return -1;
+	}
+	DEBUG(DEBUG_DEBUG, ("created pd %p\n", pconn->pd));
+
+	/* init mr */
+	if (ibw_init_memory(conn))
+		return -1;
+
+	/* init cq */
+	pconn->cq = ibv_create_cq(pconn->cm_id->verbs,
+		pctx->opts.max_recv_wr + pctx->opts.max_send_wr,
+		conn, pconn->verbs_channel, 0);
+	if (pconn->cq==NULL) {
+		sprintf(ibw_lasterr, "ibv_create_cq failed\n");
+		return -1;
+	}
+
+	rc = ibv_req_notify_cq(pconn->cq, 0);
+	if (rc) {
+		sprintf(ibw_lasterr, "ibv_req_notify_cq failed with %d\n", rc);
+		return rc;
+	}
+
+	/* init qp */
+	memset(&init_attr, 0, sizeof(init_attr));
+	init_attr.cap.max_send_wr = pctx->opts.max_send_wr;
+	init_attr.cap.max_recv_wr = pctx->opts.max_recv_wr;
+	init_attr.cap.max_recv_sge = 1;
+	init_attr.cap.max_send_sge = 1;
+	init_attr.qp_type = IBV_QPT_RC;
+	init_attr.send_cq = pconn->cq;
+	init_attr.recv_cq = pconn->cq;
+
+	rc = rdma_create_qp(pconn->cm_id, pconn->pd, &init_attr);
+	if (rc) {
+		sprintf(ibw_lasterr, "rdma_create_qp failed with %d\n", rc);
+		return rc;
+	}
+	/* elase result is in pconn->cm_id->qp */
+
+	rc = ibv_query_qp(pconn->cm_id->qp, &attr, IBV_QP_PATH_MTU, &init_attr);
+	if (rc) {
+		sprintf(ibw_lasterr, "ibv_query_qp failed with %d\n", rc);
+		return rc;
+	}
+
+	return ibw_fill_cq(conn);
+}
+
+static int ibw_refill_cq_recv(struct ibw_conn *conn)
+{
+	struct ibw_ctx_priv *pctx = talloc_get_type(conn->ctx->internal, struct ibw_ctx_priv);
+	struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
+	int	rc;
+	struct ibv_sge list = {
+		.addr 	= (uintptr_t) NULL, /* filled below */
+		.length = pctx->opts.recv_bufsize,
+		.lkey 	= pconn->mr_recv->lkey /* always the same */
+	};
+	struct ibv_recv_wr wr = {
+		.wr_id 	    = 0, /* filled below */
+		.sg_list    = &list,
+		.num_sge    = 1,
+	};
+	struct ibv_recv_wr *bad_wr;
+
+	DEBUG(DEBUG_DEBUG, ("ibw_refill_cq_recv(cmid: %p)\n", pconn->cm_id));
+
+	list.addr = (uintptr_t) pconn->buf_recv + pctx->opts.recv_bufsize * pconn->recv_index;
+	wr.wr_id = pconn->recv_index;
+	pconn->recv_index = (pconn->recv_index + 1) % pctx->opts.max_recv_wr;
+
+	rc = ibv_post_recv(pconn->cm_id->qp, &wr, &bad_wr);
+	if (rc) {
+		sprintf(ibw_lasterr, "refill/ibv_post_recv failed with %d\n", rc);
+		DEBUG(DEBUG_ERR, ("%s", ibw_lasterr));
+		return -2;
+	}
+
+	return 0;
+}
+
+static int ibw_fill_cq(struct ibw_conn *conn)
+{
+	struct ibw_ctx_priv *pctx = talloc_get_type(conn->ctx->internal, struct ibw_ctx_priv);
+	struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
+	int	i, rc;
+	struct ibv_sge list = {
+		.addr 	= (uintptr_t) NULL, /* filled below */
+		.length = pctx->opts.recv_bufsize,
+		.lkey 	= pconn->mr_recv->lkey /* always the same */
+	};
+	struct ibv_recv_wr wr = {
+		.wr_id 	    = 0, /* filled below */
+		.sg_list    = &list,
+		.num_sge    = 1,
+	};
+	struct ibv_recv_wr *bad_wr;
+
+	DEBUG(DEBUG_DEBUG, ("ibw_fill_cq(cmid: %p)\n", pconn->cm_id));
+
+	for(i = pctx->opts.max_recv_wr; i!=0; i--) {
+		list.addr = (uintptr_t) pconn->buf_recv + pctx->opts.recv_bufsize * pconn->recv_index;
+		wr.wr_id = pconn->recv_index;
+		pconn->recv_index = (pconn->recv_index + 1) % pctx->opts.max_recv_wr;
+
+		rc = ibv_post_recv(pconn->cm_id->qp, &wr, &bad_wr);
+		if (rc) {
+			sprintf(ibw_lasterr, "fill/ibv_post_recv failed with %d\n", rc);
+			DEBUG(DEBUG_ERR, ("%s", ibw_lasterr));
+			return -2;
+		}
+	}
+
+	return 0;
+}
+
+static int ibw_manage_connect(struct ibw_conn *conn)
+{
+	struct rdma_conn_param conn_param;
+	struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
+	int	rc;
+
+	DEBUG(DEBUG_DEBUG, ("ibw_manage_connect(cmid: %p)\n", pconn->cm_id));
+
+	if (ibw_setup_cq_qp(conn))
+		return -1;
+
+	/* cm connect */
+	memset(&conn_param, 0, sizeof conn_param);
+	conn_param.responder_resources = 1;
+	conn_param.initiator_depth = 1;
+	conn_param.retry_count = 10;
+
+	rc = rdma_connect(pconn->cm_id, &conn_param);
+	if (rc)
+		sprintf(ibw_lasterr, "rdma_connect error %d\n", rc);
+
+	return rc;
+}
+
+static void ibw_event_handler_cm(struct tevent_context *ev,
+	struct tevent_fd *fde, uint16_t flags, void *private_data)
+{
+	int	rc;
+	struct ibw_ctx	*ctx = talloc_get_type(private_data, struct ibw_ctx);
+	struct ibw_ctx_priv *pctx = talloc_get_type(ctx->internal, struct ibw_ctx_priv);
+	struct ibw_conn *conn = NULL;
+	struct ibw_conn_priv *pconn = NULL;
+	struct rdma_cm_id *cma_id = NULL;
+	struct rdma_cm_event *event = NULL;
+
+	assert(ctx!=NULL);
+
+	rc = rdma_get_cm_event(pctx->cm_channel, &event);
+	if (rc) {
+		ctx->state = IBWS_ERROR;
+		event = NULL;
+		sprintf(ibw_lasterr, "rdma_get_cm_event error %d\n", rc);
+		goto error;
+	}
+	cma_id = event->id;
+
+	DEBUG(DEBUG_DEBUG, ("cma_event type %d cma_id %p (%s)\n", event->event, cma_id,
+		  (cma_id == pctx->cm_id) ? "parent" : "child"));
+
+	switch (event->event) {
+	case RDMA_CM_EVENT_ADDR_RESOLVED:
+		DEBUG(DEBUG_DEBUG, ("RDMA_CM_EVENT_ADDR_RESOLVED\n"));
+		/* continuing from ibw_connect ... */
+		rc = rdma_resolve_route(cma_id, 2000);
+		if (rc) {
+			sprintf(ibw_lasterr, "rdma_resolve_route error %d\n", rc);
+			goto error;
+		}
+		/* continued at RDMA_CM_EVENT_ROUTE_RESOLVED */
+		break;
+
+	case RDMA_CM_EVENT_ROUTE_RESOLVED:
+		DEBUG(DEBUG_DEBUG, ("RDMA_CM_EVENT_ROUTE_RESOLVED\n"));
+		/* after RDMA_CM_EVENT_ADDR_RESOLVED: */
+		assert(cma_id->context!=NULL);
+		conn = talloc_get_type(cma_id->context, struct ibw_conn);
+
+		rc = ibw_manage_connect(conn);
+		if (rc)
+			goto error;
+
+		break;
+
+	case RDMA_CM_EVENT_CONNECT_REQUEST:
+		DEBUG(DEBUG_DEBUG, ("RDMA_CM_EVENT_CONNECT_REQUEST\n"));
+		ctx->state = IBWS_CONNECT_REQUEST;
+		conn = ibw_conn_new(ctx, ctx);
+		pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
+		pconn->cm_id = cma_id; /* !!! event will be freed but id not */
+		cma_id->context = (void *)conn;
+		DEBUG(DEBUG_DEBUG, ("pconn->cm_id %p\n", pconn->cm_id));
+
+		if (ibw_setup_cq_qp(conn))
+			goto error;
+
+		conn->state = IBWC_INIT;
+		pctx->connstate_func(ctx, conn);
+
+		/* continued at ibw_accept when invoked by the func above */
+		if (!pconn->is_accepted) {
+			rc = rdma_reject(cma_id, NULL, 0);
+			if (rc)
+				DEBUG(DEBUG_ERR, ("rdma_reject failed with rc=%d\n", rc));
+			talloc_free(conn);
+			DEBUG(DEBUG_DEBUG, ("pconn->cm_id %p wasn't accepted\n", pconn->cm_id));
+		}
+
+		/* TODO: clarify whether if it's needed by upper layer: */
+		ctx->state = IBWS_READY;
+		pctx->connstate_func(ctx, NULL);
+
+		/* NOTE: more requests can arrive until RDMA_CM_EVENT_ESTABLISHED ! */
+		break;
+
+	case RDMA_CM_EVENT_ESTABLISHED:
+		/* expected after ibw_accept and ibw_connect[not directly] */
+		DEBUG(DEBUG_INFO, ("ESTABLISHED (conn: %p)\n", cma_id->context));
+		conn = talloc_get_type(cma_id->context, struct ibw_conn);
+		assert(conn!=NULL); /* important assumption */
+
+		DEBUG(DEBUG_DEBUG, ("ibw_setup_cq_qp succeeded (cmid=%p)\n", cma_id));
+
+		/* client conn is up */
+		conn->state = IBWC_CONNECTED;
+
+		/* both ctx and conn have changed */
+		pctx->connstate_func(ctx, conn);
+		break;
+
+	case RDMA_CM_EVENT_ADDR_ERROR:
+		sprintf(ibw_lasterr, "RDMA_CM_EVENT_ADDR_ERROR, error %d\n", event->status);
+		goto error;
+	case RDMA_CM_EVENT_ROUTE_ERROR:
+		sprintf(ibw_lasterr, "RDMA_CM_EVENT_ROUTE_ERROR, error %d\n", event->status);
+		goto error;
+	case RDMA_CM_EVENT_CONNECT_ERROR:
+		sprintf(ibw_lasterr, "RDMA_CM_EVENT_CONNECT_ERROR, error %d\n", event->status);
+		goto error;
+	case RDMA_CM_EVENT_UNREACHABLE:
+		sprintf(ibw_lasterr, "RDMA_CM_EVENT_UNREACHABLE, error %d\n", event->status);
+		goto error;
+	case RDMA_CM_EVENT_REJECTED:
+		sprintf(ibw_lasterr, "RDMA_CM_EVENT_REJECTED, error %d\n", event->status);
+		DEBUG(DEBUG_INFO, ("cm event handler: %s", ibw_lasterr));
+		conn = talloc_get_type(cma_id->context, struct ibw_conn);
+		if (conn) {
+			/* must be done BEFORE connstate */
+			if ((rc=rdma_ack_cm_event(event)))
+				DEBUG(DEBUG_ERR, ("reject/rdma_ack_cm_event failed with %d\n", rc));
+			event = NULL; /* not to touch cma_id or conn */
+			conn->state = IBWC_ERROR;
+			/* it should free the conn */
+			pctx->connstate_func(NULL, conn);
+		}
+		break; /* this is not strictly an error */
+
+	case RDMA_CM_EVENT_DISCONNECTED:
+		DEBUG(DEBUG_DEBUG, ("RDMA_CM_EVENT_DISCONNECTED\n"));
+		if ((rc=rdma_ack_cm_event(event)))
+			DEBUG(DEBUG_ERR, ("disc/rdma_ack_cm_event failed with %d\n", rc));
+		event = NULL; /* don't ack more */
+
+		if (cma_id!=pctx->cm_id) {
+			DEBUG(DEBUG_ERR, ("client DISCONNECT event cm_id=%p\n", cma_id));
+			conn = talloc_get_type(cma_id->context, struct ibw_conn);
+			conn->state = IBWC_DISCONNECTED;
+			pctx->connstate_func(NULL, conn);
+		}
+		break;
+
+	case RDMA_CM_EVENT_DEVICE_REMOVAL:
+		sprintf(ibw_lasterr, "cma detected device removal!\n");
+		goto error;
+
+	default:
+		sprintf(ibw_lasterr, "unknown event %d\n", event->event);
+		goto error;
+	}
+
+	if (event!=NULL && (rc=rdma_ack_cm_event(event))) {
+		sprintf(ibw_lasterr, "rdma_ack_cm_event failed with %d\n", rc);
+		goto error;
+	}
+
+	return;
+error:
+	DEBUG(DEBUG_ERR, ("cm event handler: %s", ibw_lasterr));
+
+	if (event!=NULL) {
+		if (cma_id!=NULL && cma_id!=pctx->cm_id) {
+			conn = talloc_get_type(cma_id->context, struct ibw_conn);
+			if (conn) {
+				conn->state = IBWC_ERROR;
+				pctx->connstate_func(NULL, conn);
+			}
+		} else {
+			ctx->state = IBWS_ERROR;
+			pctx->connstate_func(ctx, NULL);
+		}
+
+		if ((rc=rdma_ack_cm_event(event))!=0) {
+			DEBUG(DEBUG_ERR, ("rdma_ack_cm_event failed with %d\n", rc));
+		}
+	}
+
+	return;
+}
+
+static void ibw_event_handler_verbs(struct tevent_context *ev,
+	struct tevent_fd *fde, uint16_t flags, void *private_data)
+{
+	struct ibw_conn	*conn = talloc_get_type(private_data, struct ibw_conn);
+	struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
+	struct ibw_ctx_priv *pctx = talloc_get_type(conn->ctx->internal, struct ibw_ctx_priv);
+
+	struct ibv_wc wc;
+	int rc;
+	struct ibv_cq *ev_cq;
+	void          *ev_ctx;
+
+	DEBUG(DEBUG_DEBUG, ("ibw_event_handler_verbs(%u)\n", (uint32_t)flags));
+
+	/* TODO: check whether if it's good to have more channels here... */
+	rc = ibv_get_cq_event(pconn->verbs_channel, &ev_cq, &ev_ctx);
+	if (rc) {
+		sprintf(ibw_lasterr, "Failed to get cq_event with %d\n", rc);
+		goto error;
+	}
+	if (ev_cq != pconn->cq) {
+		sprintf(ibw_lasterr, "ev_cq(%p) != pconn->cq(%p)\n", ev_cq, pconn->cq);
+		goto error;
+	}
+	rc = ibv_req_notify_cq(pconn->cq, 0);
+	if (rc) {
+		sprintf(ibw_lasterr, "Couldn't request CQ notification (%d)\n", rc);
+		goto error;
+	}
+
+	while((rc=ibv_poll_cq(pconn->cq, 1, &wc))==1) {
+		if (wc.status) {
+			sprintf(ibw_lasterr, "cq completion failed status=%d, opcode=%d, rc=%d\n",
+				wc.status, wc.opcode, rc);
+			goto error;
+		}
+
+		switch(wc.opcode) {
+		case IBV_WC_SEND:
+			DEBUG(DEBUG_DEBUG, ("send completion\n"));
+			if (ibw_wc_send(conn, &wc))
+				goto error;
+			break;
+
+		case IBV_WC_RDMA_WRITE:
+			DEBUG(DEBUG_DEBUG, ("rdma write completion\n"));
+			break;
+	
+		case IBV_WC_RDMA_READ:
+			DEBUG(DEBUG_DEBUG, ("rdma read completion\n"));
+			break;
+
+		case IBV_WC_RECV:
+			DEBUG(DEBUG_DEBUG, ("recv completion\n"));
+			if (ibw_wc_recv(conn, &wc))
+				goto error;
+			break;
+
+		default:
+			sprintf(ibw_lasterr, "unknown completion %d\n", wc.opcode);
+			goto error;
+		}
+	}
+	if (rc!=0) {
+		sprintf(ibw_lasterr, "ibv_poll_cq error %d\n", rc);
+		goto error;
+	}
+
+	ibv_ack_cq_events(pconn->cq, 1);
+
+	return;
+error:
+	ibv_ack_cq_events(pconn->cq, 1);
+
+	DEBUG(DEBUG_ERR, ("%s", ibw_lasterr));
+	
+	if (conn->state!=IBWC_ERROR) {
+		conn->state = IBWC_ERROR;
+		pctx->connstate_func(NULL, conn);
+	}
+}
+
+static int ibw_process_queue(struct ibw_conn *conn)
+{
+	struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
+	struct ibw_ctx_priv *pctx;
+	struct ibw_wr	*p;
+	int	rc;
+	uint32_t	msg_size;
+
+	if (pconn->queue==NULL)
+		return 0; /* NOP */
+
+	p = pconn->queue;
+
+	/* we must have at least 1 fragment to send */
+	assert(p->queued_ref_cnt>0);
+	p->queued_ref_cnt--;
+
+	pctx = talloc_get_type(conn->ctx->internal, struct ibw_ctx_priv);
+	msg_size = (p->queued_ref_cnt) ? pctx->opts.recv_bufsize : p->queued_rlen;
+
+	assert(p->queued_msg!=NULL);
+	assert(msg_size!=0);
+
+	DEBUG(DEBUG_DEBUG, ("ibw_process_queue refcnt=%d msgsize=%u\n",
+		p->queued_ref_cnt, msg_size));
+
+	rc = ibw_send_packet(conn, p->queued_msg, p, msg_size);
+
+	/* was this the last fragment? */
+	if (p->queued_ref_cnt) {
+		p->queued_msg += pctx->opts.recv_bufsize;
+	} else {
+		DLIST_REMOVE2(pconn->queue, p, qprev, qnext);
+		p->queued_msg = NULL;
+	}
+
+	return rc;
+}
+
+static int ibw_wc_send(struct ibw_conn *conn, struct ibv_wc *wc)
+{
+	struct ibw_ctx_priv *pctx = talloc_get_type(conn->ctx->internal, struct ibw_ctx_priv);
+	struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
+	struct ibw_wr	*p;
+	int	send_index;
+
+	DEBUG(DEBUG_DEBUG, ("ibw_wc_send(cmid: %p, wr_id: %u, bl: %u)\n",
+		pconn->cm_id, (uint32_t)wc->wr_id, (uint32_t)wc->byte_len));
+
+	assert(pconn->cm_id->qp->qp_num==wc->qp_num);
+	assert(wc->wr_id >= pctx->opts.max_recv_wr);
+	send_index = wc->wr_id - pctx->opts.max_recv_wr;
+	pconn->wr_sent--;
+
+	if (send_index < pctx->opts.max_send_wr) {
+		DEBUG(DEBUG_DEBUG, ("ibw_wc_send#1 %u\n", (int)wc->wr_id));
+		p = pconn->wr_index[send_index];
+		if (p->buf_large!=NULL) {
+			if (p->ref_cnt) {
+				/* awaiting more of it... */
+				p->ref_cnt--;
+			} else {
+				ibw_free_mr(&p->buf_large, &p->mr_large);
+				DLIST_REMOVE(pconn->wr_list_used, p);
+				DLIST_ADD(pconn->wr_list_avail, p);
+			}
+		} else { /* nasty - but necessary */
+			DLIST_REMOVE(pconn->wr_list_used, p);
+			DLIST_ADD(pconn->wr_list_avail, p);
+		}
+	} else { /* "extra" request - not optimized */
+		DEBUG(DEBUG_DEBUG, ("ibw_wc_send#2 %u\n", (int)wc->wr_id));
+		for(p=pconn->extra_sent; p!=NULL; p=p->next)
+			if ((p->wr_id + pctx->opts.max_recv_wr)==(int)wc->wr_id)
+				break;
+		if (p==NULL) {
+			sprintf(ibw_lasterr, "failed to find wr_id %d\n", (int)wc->wr_id);
+				return -1;
+		}
+		if (p->ref_cnt) {
+			p->ref_cnt--;
+		} else {
+			ibw_free_mr(&p->buf_large, &p->mr_large);
+			DLIST_REMOVE(pconn->extra_sent, p);
+			DLIST_ADD(pconn->extra_avail, p);
+		}
+	}
+
+	return ibw_process_queue(conn);
+}
+
+static int ibw_append_to_part(struct ibw_conn_priv *pconn,
+	struct ibw_part *part, char **pp, uint32_t add_len, int info)
+{
+	DEBUG(DEBUG_DEBUG, ("ibw_append_to_part: cmid=%p, (bs=%u, len=%u, tr=%u), al=%u, i=%u\n",
+		pconn->cm_id, part->bufsize, part->len, part->to_read, add_len, info));
+
+	/* allocate more if necessary - it's an "evergrowing" buffer... */
+	if (part->len + add_len > part->bufsize) {
+		if (part->buf==NULL) {
+			assert(part->len==0);
+			part->buf = talloc_size(pconn, add_len);
+			if (part->buf==NULL) {
+				sprintf(ibw_lasterr, "recv talloc_size error (%u) #%d\n",
+					add_len, info);
+				return -1;
+			}
+			part->bufsize = add_len;
+		} else {
+			part->buf = talloc_realloc_size(pconn,
+				part->buf, part->len + add_len);
+			if (part->buf==NULL) {
+				sprintf(ibw_lasterr, "recv realloc error (%u + %u) #%d\n",
+					part->len, add_len, info);
+				return -1;
+			}
+		}
+		part->bufsize = part->len + add_len;
+	}
+
+	/* consume pp */
+	memcpy(part->buf + part->len, *pp, add_len);
+	*pp += add_len;
+	part->len += add_len;
+	part->to_read -= add_len;
+
+	return 0;
+}
+
+static int ibw_wc_mem_threshold(struct ibw_conn_priv *pconn,
+	struct ibw_part *part, uint32_t threshold)
+{
+	DEBUG(DEBUG_DEBUG, ("ibw_wc_mem_threshold: cmid=%p, (bs=%u, len=%u, tr=%u), thr=%u\n",
+		pconn->cm_id, part->bufsize, part->len, part->to_read, threshold));
+
+	if (part->bufsize > threshold) {
+		DEBUG(DEBUG_DEBUG, ("ibw_wc_mem_threshold: cmid=%p, %u > %u\n",
+			pconn->cm_id, part->bufsize, threshold));
+		talloc_free(part->buf);
+		part->buf = talloc_size(pconn, threshold);
+		if (part->buf==NULL) {
+			sprintf(ibw_lasterr, "talloc_size failed\n");
+			return -1;
+		}
+		part->bufsize = threshold;
+	}
+	return 0;
+}
+
+static int ibw_wc_recv(struct ibw_conn *conn, struct ibv_wc *wc)
+{
+	struct ibw_ctx_priv *pctx = talloc_get_type(conn->ctx->internal, struct ibw_ctx_priv);
+	struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
+	struct ibw_part	*part = &pconn->part;
+	char	*p;
+	uint32_t	remain = wc->byte_len;
+
+	DEBUG(DEBUG_DEBUG, ("ibw_wc_recv: cmid=%p, wr_id: %u, bl: %u\n",
+		pconn->cm_id, (uint32_t)wc->wr_id, remain));
+
+	assert(pconn->cm_id->qp->qp_num==wc->qp_num);
+	assert((int)wc->wr_id < pctx->opts.max_recv_wr);
+	assert(wc->byte_len <= pctx->opts.recv_bufsize);
+
+	p = pconn->buf_recv + ((int)wc->wr_id * pctx->opts.recv_bufsize);
+
+	while(remain) {
+		/* here always true: (part->len!=0 && part->to_read!=0) ||
+			(part->len==0 && part->to_read==0) */
+		if (part->len) { /* is there a partial msg to be continued? */
+			int read_len = (part->to_read<=remain) ? part->to_read : remain;
+			if (ibw_append_to_part(pconn, part, &p, read_len, 421))
+				goto error;
+			remain -= read_len;
+
+			if (part->len<=sizeof(uint32_t) && part->to_read==0) {
+				assert(part->len==sizeof(uint32_t));
+				/* set it again now... */
+				part->to_read = *((uint32_t *)(part->buf)); /* TODO: ntohl */
+				if (part->to_read<sizeof(uint32_t)) {
+					sprintf(ibw_lasterr, "got msglen=%u #2\n", part->to_read);
+					goto error;
+				}
+				part->to_read -= sizeof(uint32_t); /* it's already read */
+			}
+
+			if (part->to_read==0) {
+				if (pctx->receive_func(conn, part->buf, part->len) != 0) {
+					goto error;
+				}
+				part->len = 0; /* tells not having partial data (any more) */
+				if (ibw_wc_mem_threshold(pconn, part, pctx->opts.recv_threshold))
+					goto error;
+			}
+		} else {
+			if (remain>=sizeof(uint32_t)) {
+				uint32_t msglen = *(uint32_t *)p; /* TODO: ntohl */
+				if (msglen<sizeof(uint32_t)) {
+					sprintf(ibw_lasterr, "got msglen=%u\n", msglen);
+					goto error;
+				}
+
+				/* mostly awaited case: */
+				if (msglen<=remain) {
+					if (pctx->receive_func(conn, p, msglen) != 0) {
+						goto error;
+					}
+					p += msglen;
+					remain -= msglen;
+				} else {
+					part->to_read = msglen;
+					/* part->len is already 0 */
+					if (ibw_append_to_part(pconn, part, &p, remain, 422))
+						goto error;
+					remain = 0; /* to be continued ... */
+					/* part->to_read > 0 here */
+				}
+			} else { /* edge case: */
+				part->to_read = sizeof(uint32_t);
+				/* part->len is already 0 */
+				if (ibw_append_to_part(pconn, part, &p, remain, 423))
+					goto error;
+				remain = 0;
+				/* part->to_read > 0 here */
+			}
+		}
+	} /* <remain> is always decreased at least by 1 */
+
+	if (ibw_refill_cq_recv(conn))
+		goto error;
+
+	return 0;
+
+error:
+	DEBUG(DEBUG_ERR, ("ibw_wc_recv error: %s", ibw_lasterr));
+	return -1;
+}
+
+static int ibw_process_init_attrs(struct ibw_initattr *attr, int nattr, struct ibw_opts *opts)
+{
+	int	i;
+	const char *name, *value;
+
+	DEBUG(DEBUG_DEBUG, ("ibw_process_init_attrs: nattr: %d\n", nattr));
+
+	opts->max_send_wr = IBW_MAX_SEND_WR;
+	opts->max_recv_wr = IBW_MAX_RECV_WR;
+	opts->recv_bufsize = IBW_RECV_BUFSIZE;
+	opts->recv_threshold = IBW_RECV_THRESHOLD;
+
+	for(i=0; i<nattr; i++) {
+		name = attr[i].name;
+		value = attr[i].value;
+
+		assert(name!=NULL && value!=NULL);
+		if (strcmp(name, "max_send_wr")==0)
+			opts->max_send_wr = atoi(value);
+		else if (strcmp(name, "max_recv_wr")==0)
+			opts->max_recv_wr = atoi(value);
+		else if (strcmp(name, "recv_bufsize")==0)
+			opts->recv_bufsize = atoi(value);
+		else if (strcmp(name, "recv_threshold")==0)
+			opts->recv_threshold = atoi(value);
+		else {
+			sprintf(ibw_lasterr, "ibw_init: unknown name %s\n", name);
+			return -1;
+		}
+	}
+	return 0;
+}
+
+struct ibw_ctx *ibw_init(struct ibw_initattr *attr, int nattr,
+	void *ctx_userdata,
+	ibw_connstate_fn_t ibw_connstate,
+	ibw_receive_fn_t ibw_receive,
+	struct tevent_context *ectx)
+{
+	struct ibw_ctx *ctx = talloc_zero(NULL, struct ibw_ctx);
+	struct ibw_ctx_priv *pctx;
+	int	rc;
+
+	DEBUG(DEBUG_DEBUG, ("ibw_init(ctx_userdata: %p, ectx: %p)\n", ctx_userdata, ectx));
+
+	/* initialize basic data structures */
+	memset(ibw_lasterr, 0, IBW_LASTERR_BUFSIZE);
+
+	assert(ctx!=NULL);
+	ibw_lasterr[0] = '\0';
+	talloc_set_destructor(ctx, ibw_ctx_destruct);
+	ctx->ctx_userdata = ctx_userdata;
+
+	pctx = talloc_zero(ctx, struct ibw_ctx_priv);
+	talloc_set_destructor(pctx, ibw_ctx_priv_destruct);
+	ctx->internal = (void *)pctx;
+	assert(pctx!=NULL);
+
+	pctx->connstate_func = ibw_connstate;
+	pctx->receive_func = ibw_receive;
+
+	pctx->ectx = ectx;
+
+	/* process attributes */
+	if (ibw_process_init_attrs(attr, nattr, &pctx->opts))
+		goto cleanup;
+
+	/* init cm */
+	pctx->cm_channel = rdma_create_event_channel();
+	if (!pctx->cm_channel) {
+		sprintf(ibw_lasterr, "rdma_create_event_channel error %d\n", errno);
+		goto cleanup;
+	}
+
+	pctx->cm_channel_event = tevent_add_fd(pctx->ectx, pctx,
+		pctx->cm_channel->fd, TEVENT_FD_READ, ibw_event_handler_cm, ctx);
+
+#if RDMA_USER_CM_MAX_ABI_VERSION >= 2
+	rc = rdma_create_id(pctx->cm_channel, &pctx->cm_id, ctx, RDMA_PS_TCP);
+#else
+	rc = rdma_create_id(pctx->cm_channel, &pctx->cm_id, ctx);
+#endif
+	if (rc) {
+		rc = errno;
+		sprintf(ibw_lasterr, "rdma_create_id error %d\n", rc);
+		goto cleanup;
+	}
+	DEBUG(DEBUG_DEBUG, ("created cm_id %p\n", pctx->cm_id));
+
+	pctx->pagesize = sysconf(_SC_PAGESIZE);
+
+	return ctx;
+	/* don't put code here */
+cleanup:
+	DEBUG(DEBUG_ERR, ("%s", ibw_lasterr));
+
+	if (ctx)
+		talloc_free(ctx);
+
+	return NULL;
+}
+
+int ibw_stop(struct ibw_ctx *ctx)
+{
+	struct ibw_ctx_priv *pctx = (struct ibw_ctx_priv *)ctx->internal;
+	struct ibw_conn *p;
+
+	DEBUG(DEBUG_DEBUG, ("ibw_stop\n"));
+
+	for(p=ctx->conn_list; p!=NULL; p=p->next) {
+		if (p->state==IBWC_ERROR || p->state==IBWC_CONNECTED) {
+			if (ibw_disconnect(p))
+				return -1;
+		}
+	}
+
+	ctx->state = IBWS_STOPPED;
+	pctx->connstate_func(ctx, NULL);
+
+	return 0;
+}
+
+int ibw_bind(struct ibw_ctx *ctx, struct sockaddr_in *my_addr)
+{
+	struct ibw_ctx_priv *pctx = (struct ibw_ctx_priv *)ctx->internal;
+	int	rc;
+
+	DEBUG(DEBUG_DEBUG, ("ibw_bind: addr=%s, port=%u\n",
+		inet_ntoa(my_addr->sin_addr), ntohs(my_addr->sin_port)));
+	rc = rdma_bind_addr(pctx->cm_id, (struct sockaddr *) my_addr);
+	if (rc) {
+		sprintf(ibw_lasterr, "rdma_bind_addr error %d\n", rc);
+		DEBUG(DEBUG_ERR, ("%s", ibw_lasterr));
+		return rc;
+	}
+	DEBUG(DEBUG_DEBUG, ("rdma_bind_addr successful\n"));
+
+	return 0;
+}
+
+int ibw_listen(struct ibw_ctx *ctx, int backlog)
+{
+	struct ibw_ctx_priv *pctx = talloc_get_type(ctx->internal, struct ibw_ctx_priv);
+	int	rc;
+
+	DEBUG(DEBUG_DEBUG, ("ibw_listen\n"));
+	rc = rdma_listen(pctx->cm_id, backlog);
+	if (rc) {
+		sprintf(ibw_lasterr, "rdma_listen failed: %d\n", rc);
+		DEBUG(DEBUG_ERR, ("%s", ibw_lasterr));
+		return rc;
+	}
+
+	return 0;
+}
+
+int ibw_accept(struct ibw_ctx *ctx, struct ibw_conn *conn, void *conn_userdata)
+{
+	struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
+	struct rdma_conn_param	conn_param;
+	int	rc;
+
+	DEBUG(DEBUG_DEBUG, ("ibw_accept: cmid=%p\n", pconn->cm_id));
+	conn->conn_userdata = conn_userdata;
+
+	memset(&conn_param, 0, sizeof(struct rdma_conn_param));
+	conn_param.responder_resources = 1;
+	conn_param.initiator_depth = 1;
+	rc = rdma_accept(pconn->cm_id, &conn_param);
+	if (rc) {
+		sprintf(ibw_lasterr, "rdma_accept failed %d\n", rc);
+		DEBUG(DEBUG_ERR, ("%s", ibw_lasterr));
+		return -1;;
+	}
+
+	pconn->is_accepted = 1;
+
+	/* continued at RDMA_CM_EVENT_ESTABLISHED */
+
+	return 0;
+}
+
+int ibw_connect(struct ibw_conn *conn, struct sockaddr_in *serv_addr, void *conn_userdata)
+{
+	struct ibw_ctx_priv *pctx = talloc_get_type(conn->ctx->internal, struct ibw_ctx_priv);
+	struct ibw_conn_priv *pconn = NULL;
+	int	rc;
+
+	assert(conn!=NULL);
+
+	conn->conn_userdata = conn_userdata;
+	pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
+	DEBUG(DEBUG_DEBUG, ("ibw_connect: addr=%s, port=%u\n", inet_ntoa(serv_addr->sin_addr),
+		ntohs(serv_addr->sin_port)));
+
+	/* clean previous - probably half - initialization */
+	if (ibw_conn_priv_destruct(pconn)) {
+		DEBUG(DEBUG_ERR, ("ibw_connect/ibw_pconn_destruct failed for cm_id=%p\n", pconn->cm_id));
+		return -1;
+	}
+
+	/* init cm */
+#if RDMA_USER_CM_MAX_ABI_VERSION >= 2
+	rc = rdma_create_id(pctx->cm_channel, &pconn->cm_id, conn, RDMA_PS_TCP);
+#else
+	rc = rdma_create_id(pctx->cm_channel, &pconn->cm_id, conn);
+#endif
+	if (rc) {
+		rc = errno;
+		sprintf(ibw_lasterr, "ibw_connect/rdma_create_id error %d\n", rc);
+		talloc_free(conn);
+		return -1;
+	}
+	DEBUG(DEBUG_DEBUG, ("ibw_connect: rdma_create_id succeeded, cm_id=%p\n", pconn->cm_id));
+
+	rc = rdma_resolve_addr(pconn->cm_id, NULL, (struct sockaddr *) serv_addr, 2000);
+	if (rc) {
+		sprintf(ibw_lasterr, "rdma_resolve_addr error %d\n", rc);
+		DEBUG(DEBUG_ERR, ("%s", ibw_lasterr));
+		talloc_free(conn);
+		return -1;
+	}
+
+	/* continued at RDMA_CM_EVENT_ADDR_RESOLVED */
+
+	return 0;
+}
+
+int ibw_disconnect(struct ibw_conn *conn)
+{
+	int	rc;
+	struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
+
+	DEBUG(DEBUG_DEBUG, ("ibw_disconnect: cmid=%p\n", pconn->cm_id));
+
+	assert(pconn!=NULL);
+
+	switch(conn->state) {
+	case IBWC_ERROR:
+		ibw_conn_priv_destruct(pconn); /* do this here right now */
+		break;
+	case IBWC_CONNECTED:
+		rc = rdma_disconnect(pconn->cm_id);
+		if (rc) {
+			sprintf(ibw_lasterr, "ibw_disconnect failed with %d\n", rc);
+			DEBUG(DEBUG_ERR, ("%s", ibw_lasterr));
+			return rc;
+		}
+		break;
+	default:
+		DEBUG(DEBUG_DEBUG, ("invalid state for disconnect: %d\n", conn->state));
+		break;
+	}
+
+	return 0;
+}
+
+int ibw_alloc_send_buf(struct ibw_conn *conn, void **buf, void **key, uint32_t len)
+{
+	struct ibw_ctx_priv *pctx = talloc_get_type(conn->ctx->internal, struct ibw_ctx_priv);
+	struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
+	struct ibw_wr *p = pconn->wr_list_avail;
+
+	if (p!=NULL) {
+		DEBUG(DEBUG_DEBUG, ("ibw_alloc_send_buf#1: cmid=%p, len=%d\n", pconn->cm_id, len));
+
+		DLIST_REMOVE(pconn->wr_list_avail, p);
+		DLIST_ADD(pconn->wr_list_used, p);
+
+		if (len <= pctx->opts.recv_bufsize) {
+			*buf = (void *)p->buf;
+		} else {
+			p->buf_large = ibw_alloc_mr(pctx, pconn, len, &p->mr_large);
+			if (p->buf_large==NULL) {
+				sprintf(ibw_lasterr, "ibw_alloc_mr#1 failed\n");
+				goto error;
+			}
+			*buf = (void *)p->buf_large;
+		}
+		/* p->wr_id is already filled in ibw_init_memory */
+	} else {
+		DEBUG(DEBUG_DEBUG, ("ibw_alloc_send_buf#2: cmid=%p, len=%d\n", pconn->cm_id, len));
+		/* not optimized */
+		p = pconn->extra_avail;
+		if (!p) {
+			p = pconn->extra_avail = talloc_zero(pconn, struct ibw_wr);
+			talloc_set_destructor(p, ibw_wr_destruct);
+			if (p==NULL) {
+				sprintf(ibw_lasterr, "talloc_zero failed (emax: %u)\n", pconn->extra_max);
+				goto error;
+			}
+			p->wr_id = pctx->opts.max_send_wr + pconn->extra_max;
+			pconn->extra_max++;
+			switch(pconn->extra_max) {
+				case 1: DEBUG(DEBUG_INFO, ("warning: queue performed\n")); break;
+				case 10: DEBUG(DEBUG_INFO, ("warning: queue reached 10\n")); break;
+				case 100: DEBUG(DEBUG_INFO, ("warning: queue reached 100\n")); break;
+				case 1000: DEBUG(DEBUG_INFO, ("warning: queue reached 1000\n")); break;
+				default: break;
+			}
+		}
+
+		p->buf_large = ibw_alloc_mr(pctx, pconn, len, &p->mr_large);
+		if (p->buf_large==NULL) {
+			sprintf(ibw_lasterr, "ibw_alloc_mr#2 failed\n");
+			goto error;
+		}
+		*buf = (void *)p->buf_large;
+
+		DLIST_REMOVE(pconn->extra_avail, p);
+		/* we don't have prepared index for this, so that
+		 * we will have to find this by wr_id later on */
+		DLIST_ADD(pconn->extra_sent, p);
+	}
+
+	*key = (void *)p;
+
+	return 0;
+error:
+	DEBUG(DEBUG_ERR, ("ibw_alloc_send_buf error: %s", ibw_lasterr));
+	return -1;
+}
+
+
+static int ibw_send_packet(struct ibw_conn *conn, void *buf, struct ibw_wr *p, uint32_t len)
+{
+	struct ibw_ctx_priv *pctx = talloc_get_type(conn->ctx->internal, struct ibw_ctx_priv);
+	struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
+	int	rc;
+
+	/* can we send it right now? */
+	if (pconn->wr_sent<pctx->opts.max_send_wr) {
+		struct ibv_send_wr *bad_wr;
+		struct ibv_sge list = {
+			.addr 	= (uintptr_t)buf,
+			.length = len,
+			.lkey 	= pconn->mr_send->lkey
+		};
+		struct ibv_send_wr wr = {
+			.wr_id 	    = p->wr_id + pctx->opts.max_recv_wr,
+			.sg_list    = &list,
+			.num_sge    = 1,
+			.opcode     = IBV_WR_SEND,
+			.send_flags = IBV_SEND_SIGNALED,
+		};
+
+		if (p->buf_large==NULL) {
+			DEBUG(DEBUG_DEBUG, ("ibw_send#normal(cmid: %p, wrid: %u, n: %d)\n",
+				pconn->cm_id, (uint32_t)wr.wr_id, len));
+		} else {
+			DEBUG(DEBUG_DEBUG, ("ibw_send#large(cmid: %p, wrid: %u, n: %d)\n",
+				pconn->cm_id, (uint32_t)wr.wr_id, len));
+			list.lkey = p->mr_large->lkey;
+		}
+
+		rc = ibv_post_send(pconn->cm_id->qp, &wr, &bad_wr);
+		if (rc) {
+			sprintf(ibw_lasterr, "ibv_post_send error %d (%d)\n",
+				rc, pconn->wr_sent);
+			goto error;
+		}
+
+		pconn->wr_sent++;
+
+		return rc;
+	} /* else put the request into our own queue: */
+
+	DEBUG(DEBUG_DEBUG, ("ibw_send#queued(cmid: %p, len: %u)\n", pconn->cm_id, len));
+
+	/* TODO: clarify how to continue when state==IBWC_STOPPED */
+
+	/* to be sent by ibw_wc_send */
+	/* regardless "normal" or [a part of] "large" packet */
+	if (!p->queued_ref_cnt) {
+		DLIST_ADD_END2(pconn->queue, p, struct ibw_wr *,
+			qprev, qnext); /* TODO: optimize */
+		p->queued_msg = buf;
+	}
+	p->queued_ref_cnt++;
+	p->queued_rlen = len; /* last wins; see ibw_wc_send */
+
+	return 0;
+error:
+	DEBUG(DEBUG_ERR, ("%s", ibw_lasterr));
+	return -1;
+}
+
+int ibw_send(struct ibw_conn *conn, void *buf, void *key, uint32_t len)
+{
+	struct ibw_ctx_priv *pctx = talloc_get_type(conn->ctx->internal, struct ibw_ctx_priv);
+	struct ibw_wr *p = talloc_get_type(key, struct ibw_wr);
+	int	rc;
+
+	assert(len>=sizeof(uint32_t));
+	assert((*((uint32_t *)buf)==len)); /* TODO: htonl */
+
+	if (len > pctx->opts.recv_bufsize) {
+		struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
+		int	rlen = len;
+		char	*packet = (char *)buf;
+		uint32_t	recv_bufsize = pctx->opts.recv_bufsize;
+
+		DEBUG(DEBUG_DEBUG, ("ibw_send#frag(cmid: %p, buf: %p, len: %u)\n",
+			pconn->cm_id, buf, len));
+
+		/* single threaded => no race here: */
+		assert(p->ref_cnt==0);
+		while(rlen > recv_bufsize) {
+			rc = ibw_send_packet(conn, packet, p, recv_bufsize);
+			if (rc)
+				return rc;
+			packet += recv_bufsize;
+			rlen -= recv_bufsize;
+			p->ref_cnt++; /* not good to have it in ibw_send_packet */
+		}
+		if (rlen) {
+			rc = ibw_send_packet(conn, packet, p, rlen);
+			p->ref_cnt++; /* not good to have it in ibw_send_packet */
+		}
+		p->ref_cnt--; /* for the same handling */
+	} else {
+		assert(p->ref_cnt==0);
+		assert(p->queued_ref_cnt==0);
+
+		rc = ibw_send_packet(conn, buf, p, len);
+	}
+	return rc;
+}
+
+int ibw_cancel_send_buf(struct ibw_conn *conn, void *buf, void *key)
+{
+	struct ibw_ctx_priv *pctx = talloc_get_type(conn->ctx->internal, struct ibw_ctx_priv);
+	struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
+	struct ibw_wr *p = talloc_get_type(key, struct ibw_wr);
+
+	assert(p!=NULL);
+	assert(buf!=NULL);
+	assert(conn!=NULL);
+
+	if (p->buf_large!=NULL)
+		ibw_free_mr(&p->buf_large, &p->mr_large);
+
+	/* parallel case */
+	if (p->wr_id < pctx->opts.max_send_wr) {
+		DEBUG(DEBUG_DEBUG, ("ibw_cancel_send_buf#1 %u", (int)p->wr_id));
+		DLIST_REMOVE(pconn->wr_list_used, p);
+		DLIST_ADD(pconn->wr_list_avail, p);
+	} else { /* "extra" packet */
+		DEBUG(DEBUG_DEBUG, ("ibw_cancel_send_buf#2 %u", (int)p->wr_id));
+		DLIST_REMOVE(pconn->extra_sent, p);
+		DLIST_ADD(pconn->extra_avail, p);
+	}
+
+	return 0;
+}
+
+const char *ibw_getLastError(void)
+{
+	return ibw_lasterr;
+}
diff --git a/ctdb/ib/ibwrapper.h b/ctdb/ib/ibwrapper.h
new file mode 100644
index 0000000..d5cdc60
--- /dev/null
+++ b/ctdb/ib/ibwrapper.h
@@ -0,0 +1,218 @@
+/*
+ * Unix SMB/CIFS implementation.
+ * Wrap Infiniband calls.
+ *
+ * Copyright (C) Sven Oehme <oehmes@de.ibm.com> 2006
+ *
+ * Major code contributions by Peter Somogyi <psomogyi@gamax.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* Server communication state */
+enum ibw_state_ctx {
+	IBWS_INIT = 0, /* ctx start - after ibw_init */
+	IBWS_READY, /* after ibw_bind & ibw_listen */
+	IBWS_CONNECT_REQUEST, /* after [IBWS_READY + incoming request] */
+		/* => [(ibw_accept)IBWS_READY | (ibw_disconnect)STOPPED | ERROR] */
+	IBWS_STOPPED, /* normal stop <= ibw_disconnect+(IBWS_READY | IBWS_CONNECT_REQUEST) */
+	IBWS_ERROR /* abnormal state; ibw_stop must be called after this */
+};
+
+/* Connection state */
+struct ibw_ctx {
+	void *ctx_userdata; /* see ibw_init */
+
+	enum ibw_state_ctx state;
+	void *internal;
+
+	struct ibw_conn *conn_list; /* 1st elem of double linked list */
+};
+
+enum ibw_state_conn {
+	IBWC_INIT = 0, /* conn start - internal state */
+	IBWC_CONNECTED, /* after ibw_accept or ibw_connect */
+	IBWC_DISCONNECTED, /* after ibw_disconnect */
+	IBWC_ERROR
+};
+
+struct ibw_conn {
+	struct ibw_ctx *ctx;
+	enum ibw_state_conn state;
+
+	void *conn_userdata; /* see ibw_connect and ibw_accept */
+	void *internal;
+
+	struct ibw_conn *prev, *next;
+};
+
+/*
+ * (name, value) pair for array param of ibw_init
+ */
+struct ibw_initattr {
+	const char *name;
+	const char *value;
+};
+
+/*
+ * Callback function definition which should inform you about
+ * connection state change
+ * This callback is invoked whenever server or client connection changes.
+ * Both <conn> and <ctx> can be NULL if their state didn't change.
+ * Return nonzero on error.
+ */
+typedef int (*ibw_connstate_fn_t)(struct ibw_ctx *ctx, struct ibw_conn *conn);
+
+/*
+ * Callback function definition which should process incoming packets
+ * This callback is invoked whenever any message arrives.
+ * Return nonzero on error.
+ *
+ * Important: you mustn't store buf pointer for later use.
+ * Process its contents before returning.
+ */
+typedef int (*ibw_receive_fn_t)(struct ibw_conn *conn, void *buf, int n);
+
+/*
+ * settings: array of (name, value) pairs
+ * where name is one of:
+ *      max_send_wr [default is 256]
+ *      max_recv_wr [default is 1024]
+ * <...>
+ *
+ * Must be called _ONCE_ for each node.
+ *
+ * max_msg_size is the maximum size of a message
+ * (max_send_wr + max_recv_wr) * max_msg_size bytes allocated per connection
+ *
+ * returns non-NULL on success
+ *
+ * talloc_free must be called for the result in IBWS_STOPPED;
+ *    it will close resources by destructor
+ *    connections(ibw_conn *) must have been closed prior talloc_free
+ */
+struct ibw_ctx *ibw_init(struct ibw_initattr *attr, int nattr,
+	void *ctx_userdata,
+	ibw_connstate_fn_t ibw_connstate,
+	ibw_receive_fn_t ibw_receive,
+	struct tevent_context *ectx);
+
+/*
+ * Must be called in states of (IBWS_ERROR, IBWS_READY, IBWS_CONNECT_REQUEST)
+ *
+ * It will send out disconnect requests and free up ibw_conn structures.
+ * The ctx->state will transit to IBWS_STOPPED after every conn are disconnected.
+ * During that time, you mustn't send/recv/disconnect any more.
+ * Only after ctx->state=IBWS_STOPPED you can talloc_free the ctx.
+ */
+int ibw_stop(struct ibw_ctx *ctx);
+
+/*************** connection initiation - like stream sockets *****/
+
+/*
+ * works like socket bind
+ * needs a normal internet address here
+ *
+ * return 0 on success
+ */
+int ibw_bind(struct ibw_ctx *ctx, struct sockaddr_in *my_addr);
+
+/*
+ * works like socket listen
+ * non-blocking
+ * enables accepting incoming connections (after IBWS_READY)
+ * (it doesn't touch ctx->state by itself)
+ *
+ * returns 0 on success
+ */
+int ibw_listen(struct ibw_ctx *ctx, int backlog);
+
+/*
+ * works like socket accept
+ * initializes a connection to a client
+ * must be called when state=IBWS_CONNECT_REQUEST
+ *
+ * returns 0 on success
+ *
+ * You have +1 waiting here: you will get ibw_conn (having the
+ * same <conn_userdata> member) structure in ibw_connstate_fn_t.
+ *
+ * Important: you won't get remote IP address (only internal conn info)
+ */
+int ibw_accept(struct ibw_ctx *ctx, struct ibw_conn *conn, void *conn_userdata);
+
+/*
+ * Create a new connection structure
+ * available for queueing ibw_send
+ *
+ * <parent> is needed to be notified by talloc destruct action.
+ */
+struct ibw_conn *ibw_conn_new(struct ibw_ctx *ctx, TALLOC_CTX *mem_ctx);
+
+/*
+ * Needs a normal internet address here
+ * can be called within IBWS_READY|IBWS_CONNECT_REQUEST
+ *
+ * returns non-NULL on success
+ *
+ * You have +1 waiting here: you will get ibw_conn (having the
+ * same <conn_userdata> member) structure in ibw_connstate_fn_t.
+ */
+int ibw_connect(struct ibw_conn *conn, struct sockaddr_in *serv_addr, void *conn_userdata);
+
+/*
+ * Sends out a disconnect request.
+ * You should process fds after calling this function
+ * and then process it with ibw_process_event normally
+ * until you get conn->state = IBWC_DISCONNECTED
+ *
+ * You mustn't talloc_free <conn> yet right after this,
+ * first wait for IBWC_DISCONNECTED.
+ */
+int ibw_disconnect(struct ibw_conn *conn);
+
+/************ Infiniband specific event loop wrapping ******************/
+
+/*
+ * You have to use this buf to fill in before send.
+ * It's just to avoid memcpy.in ibw_send.
+ * Use the same (buf, key) pair with ibw_send.
+ * Don't use more space than maxsize (see ibw_init).
+ *
+ * Returns 0 on success.
+ */
+int ibw_alloc_send_buf(struct ibw_conn *conn, void **buf, void **key, uint32_t len);
+
+/*
+ * Send the message in one
+ * Can be invoked any times (should fit into buffers) and at any time
+ * (in conn->state=IBWC_CONNECTED)
+ * n must be less or equal than max_msg_size (see ibw_init)
+ *
+ * You mustn't use (buf, key) any more for sending.
+ */
+int ibw_send(struct ibw_conn *conn, void *buf, void *key, uint32_t len);
+
+/*
+ * Call this after ibw_alloc_send_buf
+ * when you won't call ibw_send for (buf, key)
+ * You mustn't use (buf, key) any more.
+ */
+int ibw_cancel_send_buf(struct ibw_conn *conn, void *buf, void *key);
+
+/*
+ * Retrieves the last error
+ * result: always non-zero, mustn't be freed (static)
+ */
+const char *ibw_getLastError(void);
diff --git a/ctdb/ib/ibwrapper_internal.h b/ctdb/ib/ibwrapper_internal.h
new file mode 100644
index 0000000..b8100a8
--- /dev/null
+++ b/ctdb/ib/ibwrapper_internal.h
@@ -0,0 +1,126 @@
+/*
+ * Unix SMB/CIFS implementation.
+ * Wrap Infiniband calls.
+ *
+ * Copyright (C) Sven Oehme <oehmes@de.ibm.com> 2006
+ *
+ * Major code contributions by Peter Somogyi <psomogyi@gamax.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+struct ibw_opts {
+	uint32_t	max_send_wr;
+	uint32_t	max_recv_wr;
+	uint32_t	recv_bufsize;
+	uint32_t	recv_threshold;
+};
+
+struct ibw_wr {
+	char	*buf; /* initialized in ibw_init_memory once per connection */
+	int	wr_id; /* position in wr_index list; also used as wr id */
+
+	char	*buf_large; /* allocated specially for "large" message */
+	struct ibv_mr *mr_large;
+	int	ref_cnt; /* reference count for ibw_wc_send to know when to release */
+
+	char	*queued_msg; /* set at ibw_send - can be different than above */
+	int	queued_ref_cnt; /* instead of adding the same to the queue again */
+	uint32_t	queued_rlen; /* last wins when queued_ref_cnt>0; or simple msg size */
+
+	struct ibw_wr *next, *prev; /* in wr_list_avail or wr_list_used */
+				/* or extra_sent or extra_avail */
+	struct ibw_wr *qnext, *qprev; /* in queue */
+};
+
+struct ibw_ctx_priv {
+	struct tevent_context *ectx;
+
+	struct ibw_opts opts;
+
+	struct rdma_cm_id	*cm_id; /* server cm id */
+
+	struct rdma_event_channel *cm_channel;
+	struct tevent_fd *cm_channel_event;
+
+	ibw_connstate_fn_t connstate_func; /* see ibw_init */
+	ibw_receive_fn_t receive_func; /* see ibw_init */
+
+	long	pagesize; /* sysconf result for memalign */
+};
+
+struct ibw_part {
+	char *buf; /* talloced memory buffer */
+	uint32_t bufsize; /* allocated size of buf - always grows */
+	uint32_t len; /* message part length */
+	uint32_t to_read; /* 4 or *((uint32_t)buf) if len>=sizeof(uint32_t) */
+};
+
+struct ibw_conn_priv {
+	struct ibv_comp_channel *verbs_channel;
+	struct tevent_fd *verbs_channel_event;
+
+	struct rdma_cm_id *cm_id; /* client's cm id */
+	struct ibv_pd	*pd;
+	int	is_accepted;
+
+	struct ibv_cq	*cq; /* qp is in cm_id */
+
+	char *buf_send; /* max_send_wr * avg_send_size */
+	struct ibv_mr *mr_send;
+	struct ibw_wr *wr_list_avail;
+	struct ibw_wr *wr_list_used;
+	struct ibw_wr **wr_index; /* array[0..(qsize-1)] of (ibw_wr *) */
+	int	wr_sent; /* # of send wrs in the CQ */
+
+	struct ibw_wr *extra_sent;
+	struct ibw_wr *extra_avail;
+	int	extra_max; /* max wr_id in the queue */
+
+	struct ibw_wr *queue;
+
+	/* buf_recv is a ring buffer */
+	char *buf_recv; /* max_recv_wr * avg_recv_size */
+	struct ibv_mr *mr_recv;
+	int recv_index; /* index of the next recv buffer when refilling */
+	struct ibw_part part;
+};
+
+/* remove an element from a list - element doesn't have to be in list. */
+#define DLIST_REMOVE2(list, p, prev, next) \
+do { \
+	if ((p) == (list)) { \
+		(list) = (p)->next; \
+		if (list) (list)->prev = NULL; \
+	} else { \
+		if ((p)->prev) (p)->prev->next = (p)->next; \
+		if ((p)->next) (p)->next->prev = (p)->prev; \
+	} \
+	if ((p) != (list)) (p)->next = (p)->prev = NULL; \
+} while (0)
+
+/* hook into the end of the list - needs a tmp pointer */
+#define DLIST_ADD_END2(list, p, type, prev, next) \
+do { \
+		if (!(list)) { \
+			(list) = (p); \
+			(p)->next = (p)->prev = NULL; \
+		} else { \
+			type tmp; \
+			for (tmp = (list); tmp->next; tmp = tmp->next) ; \
+			tmp->next = (p); \
+			(p)->next = NULL; \
+			(p)->prev = tmp; \
+		} \
+} while (0)
diff --git a/ctdb/ib/ibwrapper_test.c b/ctdb/ib/ibwrapper_test.c
new file mode 100644
index 0000000..77a5323
--- /dev/null
+++ b/ctdb/ib/ibwrapper_test.c
@@ -0,0 +1,659 @@
+/*
+ * Unix SMB/CIFS implementation.
+ * Test the infiniband wrapper.
+ *
+ * Copyright (C) Sven Oehme <oehmes@de.ibm.com> 2006
+ *
+ * Major code contributions by Peter Somogyi <psomogyi@gamax.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "replace.h"
+#include "system/network.h"
+#include "system/time.h"
+#include "system/wait.h"
+
+#include <assert.h>
+#include <talloc.h>
+#include <tevent.h>
+
+#include "lib/util/time.h"
+#include "lib/util/debug.h"
+
+#include "common/logging.h"
+
+#include "ib/ibwrapper.h"
+
+struct ibwtest_ctx {
+	int	is_server;
+	char	*id; /* my id */
+
+	struct ibw_initattr *attrs;
+	int	nattrs;
+	char	*opts; /* option string */
+
+	struct sockaddr_in *addrs; /* dynamic array of dest addrs */
+	int	naddrs;
+
+	unsigned int	nsec; /* delta times between messages in nanosec */
+	unsigned int	sleep_usec; /* microsecs to sleep in the main loop to emulate overloading */
+	uint32_t	maxsize; /* maximum variable message size */
+
+	int	cnt;
+	int	nsent;
+
+	int	nmsg; /* number of messages to send (client) */
+
+	int	kill_me;
+	int	stopping;
+	int	error;
+	struct ibw_ctx	*ibwctx;
+
+	struct timeval	start_time, end_time;
+};
+
+struct ibwtest_conn {
+	char	*id;
+};
+
+enum testopcode {
+	TESTOP_SEND_ID = 1,
+	TESTOP_SEND_TEXT = 2,
+	TESTOP_SEND_RND = 3
+};
+
+static int ibwtest_connect_everybody(struct ibwtest_ctx *tcx)
+{
+	struct ibw_conn		*conn;
+	struct ibwtest_conn	*tconn = talloc_zero(tcx, struct ibwtest_conn);
+	int	i;
+
+	for(i=0; i<tcx->naddrs; i++) {
+		conn = ibw_conn_new(tcx->ibwctx, tconn);
+		if (ibw_connect(conn, &tcx->addrs[i], tconn)) {
+			fprintf(stderr, "ibw_connect error at %d\n", i);
+			return -1;
+		}
+	}
+	DEBUG(DEBUG_DEBUG, ("sent %d connect request...\n", tcx->naddrs));
+
+	return 0;
+}
+
+static int ibwtest_send_id(struct ibw_conn *conn)
+{
+	struct ibwtest_ctx *tcx = talloc_get_type(conn->ctx->ctx_userdata, struct ibwtest_ctx);
+	char *buf;
+	void *key;
+	uint32_t	len;
+
+	DEBUG(DEBUG_DEBUG, ("ibwtest_send_id\n"));
+	len = sizeof(uint32_t)+strlen(tcx->id)+2;
+	if (ibw_alloc_send_buf(conn, (void **)&buf, &key, len)) {
+		DEBUG(DEBUG_ERR, ("send_id: ibw_alloc_send_buf failed\n"));
+		return -1;
+	}
+
+	/* first sizeof(uint32_t) size bytes are for length */
+	*((uint32_t *)buf) = len;
+	buf[sizeof(uint32_t)] = (char)TESTOP_SEND_ID;
+	strcpy(buf+sizeof(uint32_t)+1, tcx->id);
+
+	if (ibw_send(conn, buf, key, len)) {
+		DEBUG(DEBUG_ERR, ("send_id: ibw_send error\n"));
+		return -1;
+	}
+	tcx->nsent++;
+
+	return 0;
+}
+
+static int ibwtest_send_test_msg(struct ibwtest_ctx *tcx, struct ibw_conn *conn, const char *msg)
+{
+	char *buf, *p;
+	void *key;
+	uint32_t len;
+
+	if (conn->state!=IBWC_CONNECTED)
+		return 0; /* not yet up */
+
+	len = strlen(msg) + 2 + sizeof(uint32_t);
+	if (ibw_alloc_send_buf(conn, (void **)&buf, &key, len)) {
+		fprintf(stderr, "send_test_msg: ibw_alloc_send_buf failed\n");
+		return -1;
+	}
+
+	*((uint32_t *)buf) = len;
+	p = buf;
+	p += sizeof(uint32_t);
+	p[0] = (char)TESTOP_SEND_TEXT;
+	p++;
+	strcpy(p, msg);
+
+	if (ibw_send(conn, buf, key, len)) {
+		DEBUG(DEBUG_ERR, ("send_test_msg: ibw_send error\n"));
+		return -1;
+	}
+	tcx->nsent++;
+
+	return 0;
+}
+
+static unsigned char ibwtest_fill_random(unsigned char *buf, uint32_t size)
+{
+	uint32_t	i = size;
+	unsigned char	sum = 0;
+	unsigned char	value;
+	while(i) {
+		i--;
+		value = (unsigned char)(256.0 * (rand() / (RAND_MAX + 1.0)));
+		buf[i] = value;
+		sum += value;
+	}
+	return sum;
+}
+
+static unsigned char ibwtest_get_sum(unsigned char *buf, uint32_t size)
+{
+	uint32_t	i = size;
+	unsigned char	sum = 0;
+
+	while(i) {
+		i--;
+		sum += buf[i];
+	}
+	return sum;
+}
+
+static int ibwtest_do_varsize_scenario_conn_size(struct ibwtest_ctx *tcx, struct ibw_conn *conn, uint32_t size)
+{
+	unsigned char *buf;
+	void	*key;
+	uint32_t	len;
+	unsigned char	sum;
+
+	len = sizeof(uint32_t) + 1 + size + 1;
+	if (ibw_alloc_send_buf(conn, (void **)&buf, &key, len)) {
+		DEBUG(DEBUG_ERR, ("varsize/ibw_alloc_send_buf failed\n"));
+		return -1;
+	}
+	*((uint32_t *)buf) = len;
+	buf[sizeof(uint32_t)] = TESTOP_SEND_RND;
+	sum = ibwtest_fill_random(buf + sizeof(uint32_t) + 1, size);
+	buf[sizeof(uint32_t) + 1 + size] = sum;
+	if (ibw_send(conn, buf, key, len)) {
+		DEBUG(DEBUG_ERR, ("varsize/ibw_send failed\n"));
+		return -1;
+	}
+	tcx->nsent++;
+
+	return 0;
+}
+
+static int ibwtest_do_varsize_scenario_conn(struct ibwtest_ctx *tcx, struct ibw_conn *conn)
+{
+	uint32_t	size;
+	int	i;
+
+	for(i=0; i<tcx->nmsg; i++)
+	{
+		//size = (uint32_t)((float)(tcx->maxsize) * (rand() / (RAND_MAX + 1.0)));
+		size = (uint32_t)((float)(tcx->maxsize) * ((float)(i+1)/(float)tcx->nmsg));
+		if (ibwtest_do_varsize_scenario_conn_size(tcx, conn, size))
+			return -1;
+	}
+	return 0;
+}
+
+/*int ibwtest_do_varsize_scenario(ibwtest_ctx *tcx)
+{
+	int	rc;
+	struct ibw_conn *conn;
+
+	for(conn=tcx->ibwctx->conn_list; conn!=NULL; conn=conn->next) {
+		if (conn->state==IBWC_CONNECTED) {
+			rc = ibwtest_do_varsize_scenario_conn(tcx, conn);
+			if (rc)
+				tcx->error = rc;
+		}
+	}
+}*/
+
+static int ibwtest_connstate_handler(struct ibw_ctx *ctx, struct ibw_conn *conn)
+{
+	struct ibwtest_ctx	*tcx = NULL; /* userdata */
+	struct ibwtest_conn	*tconn = NULL; /* userdata */
+
+	if (ctx) {
+		tcx = talloc_get_type(ctx->ctx_userdata, struct ibwtest_ctx);
+
+		switch(ctx->state) {
+		case IBWS_INIT:
+			DEBUG(DEBUG_DEBUG, ("test IBWS_INIT\n"));
+			break;
+		case IBWS_READY:
+			DEBUG(DEBUG_DEBUG, ("test IBWS_READY\n"));
+			break;
+		case IBWS_CONNECT_REQUEST:
+			DEBUG(DEBUG_DEBUG, ("test IBWS_CONNECT_REQUEST\n"));
+			tconn = talloc_zero(conn, struct ibwtest_conn);
+			if (ibw_accept(ctx, conn, tconn)) {
+				DEBUG(DEBUG_ERR, ("error accepting the connect request\n"));
+			}
+			break;
+		case IBWS_STOPPED:
+			DEBUG(DEBUG_DEBUG, ("test IBWS_STOPPED\n"));
+			tcx->kill_me = 1; /* main loop can exit */
+			break;
+		case IBWS_ERROR:
+			DEBUG(DEBUG_DEBUG, ("test IBWS_ERROR\n"));
+			ibw_stop(tcx->ibwctx);
+			break;
+		default:
+			assert(0);
+			break;
+		}
+	}
+
+	if (conn) {
+		tconn = talloc_get_type(conn->conn_userdata, struct ibwtest_conn);
+		switch(conn->state) {
+		case IBWC_INIT:
+			DEBUG(DEBUG_DEBUG, ("test IBWC_INIT\n"));
+			break;
+		case IBWC_CONNECTED:
+			if (gettimeofday(&tcx->start_time, NULL)) {
+				DEBUG(DEBUG_ERR, ("gettimeofday error %d", errno));
+				return -1;
+			}
+			ibwtest_send_id(conn);
+			break;
+		case IBWC_DISCONNECTED:
+			DEBUG(DEBUG_DEBUG, ("test IBWC_DISCONNECTED\n"));
+			talloc_free(conn);
+			break;
+		case IBWC_ERROR:
+			DEBUG(DEBUG_DEBUG, ("test IBWC_ERROR %s\n", ibw_getLastError()));
+			break;
+		default:
+			assert(0);
+			break;
+		}
+	}
+	return 0;
+}
+
+static int ibwtest_receive_handler(struct ibw_conn *conn, void *buf, int n)
+{
+	struct ibwtest_conn *tconn;
+	enum testopcode op;
+	struct ibwtest_ctx *tcx = talloc_get_type(conn->ctx->ctx_userdata, struct ibwtest_ctx);
+	int	rc = 0;
+
+	assert(conn!=NULL);
+	assert(n>=sizeof(uint32_t)+1);
+	tconn = talloc_get_type(conn->conn_userdata, struct ibwtest_conn);
+
+	op = (enum testopcode)((char *)buf)[sizeof(uint32_t)];
+	if (op==TESTOP_SEND_ID) {
+		tconn->id = talloc_strdup(tconn, ((char *)buf)+sizeof(uint32_t)+1);
+	}
+	if (op==TESTOP_SEND_ID || op==TESTOP_SEND_TEXT) {
+		DEBUG(DEBUG_DEBUG, ("[%d]msg from %s: \"%s\"(%d)\n", op,
+			tconn->id ? tconn->id : "NULL", ((char *)buf)+sizeof(uint32_t)+1, n));
+	}
+
+	if (tcx->is_server) {
+		if (op==TESTOP_SEND_RND) {
+			unsigned char sum;
+			sum = ibwtest_get_sum((unsigned char *)buf + sizeof(uint32_t) + 1,
+				n - sizeof(uint32_t) - 2);
+			DEBUG(DEBUG_DEBUG, ("[%d]msg varsize %u/sum %u from %s\n",
+				op,
+				(uint32_t)(n - sizeof(uint32_t) - 2),
+				(uint32_t)sum,
+				tconn->id ? tconn->id : "NULL"));
+			if (sum!=((unsigned char *)buf)[n-1]) {
+				DEBUG(DEBUG_ERR, ("ERROR: checksum mismatch %u!=%u\n",
+					(uint32_t)sum, (uint32_t)((unsigned char *)buf)[n-1]));
+				ibw_stop(tcx->ibwctx);
+				goto error;
+			}
+		} else if (op!=TESTOP_SEND_ID) {
+			char *buf2;
+			void *key2;
+
+			/* bounce message regardless what it is */
+			if (ibw_alloc_send_buf(conn, (void **)&buf2, &key2, n)) {
+				fprintf(stderr, "ibw_alloc_send_buf error #2\n");
+				goto error;
+			}
+			memcpy(buf2, buf, n);
+			if (ibw_send(conn, buf2, key2, n)) {
+				fprintf(stderr, "ibw_send error #2\n");
+				goto error;
+			}
+			tcx->nsent++;
+		}
+	} else { /* client: */
+		if (op==TESTOP_SEND_ID && tcx->maxsize) {
+			/* send them in one blow */
+			rc = ibwtest_do_varsize_scenario_conn(tcx, conn);
+		}
+
+		if (tcx->nmsg) {
+			char	msg[26];
+			sprintf(msg, "hello world %d", tcx->nmsg--);
+			rc = ibwtest_send_test_msg(tcx, conn, msg);
+			if (tcx->nmsg==0) {
+				ibw_stop(tcx->ibwctx);
+				tcx->stopping = 1;
+			}
+		}
+	}
+
+	if (rc)
+		tcx->error = rc;
+
+	return rc;
+error:
+	return -1;
+}
+
+static void ibwtest_timeout_handler(struct tevent_context *ev,
+				    struct tevent_timer *te,
+				    struct timeval t, void *private_data)
+{
+	struct ibwtest_ctx *tcx = talloc_get_type(private_data, struct ibwtest_ctx);
+	int	rc;
+
+	if (!tcx->is_server) {
+		struct ibw_conn *conn;
+		char	msg[50];
+
+		/* fill it with something variable... */
+		sprintf(msg, "hello world %d", tcx->cnt++);
+
+		/* send something to everybody... */
+		for(conn=tcx->ibwctx->conn_list; conn!=NULL; conn=conn->next) {
+			if (conn->state==IBWC_CONNECTED) {
+				rc = ibwtest_send_test_msg(tcx, conn, msg);
+				if (rc)
+					tcx->error = rc;
+			}
+		}
+	} /* else allow main loop run */
+}
+
+static struct ibwtest_ctx *testctx = NULL;
+
+static void ibwtest_sigint_handler(int sig)
+{
+	DEBUG(DEBUG_ERR, ("got SIGINT\n"));
+	if (testctx) {
+		if (testctx->ibwctx->state==IBWS_READY ||
+			testctx->ibwctx->state==IBWS_CONNECT_REQUEST ||
+			testctx->ibwctx->state==IBWS_ERROR)
+		{
+			if (testctx->stopping) {
+				DEBUG(DEBUG_DEBUG, ("forcing exit...\n"));
+				testctx->kill_me = 1;
+			} else {
+				/* mostly expected case */
+				ibw_stop(testctx->ibwctx);
+				testctx->stopping = 1;
+			}
+		} else
+			testctx->kill_me = 1;
+	}
+}
+
+static int ibwtest_parse_attrs(struct ibwtest_ctx *tcx, char *optext,
+	struct ibw_initattr **pattrs, int *nattrs, char op)
+{
+	int	i = 0, n = 1;
+	int	porcess_next = 1;
+	char	*p, *q;
+	struct ibw_initattr *attrs = NULL;
+
+	*pattrs = NULL;
+	for(p = optext; *p!='\0'; p++) {
+		if (*p==',')
+			n++;
+	}
+
+	attrs = (struct ibw_initattr *)talloc_size(tcx,
+		n * sizeof(struct ibw_initattr));
+	for(p = optext; *p!='\0'; p++) {
+		if (porcess_next) {
+			attrs[i].name = p;
+			q = strchr(p, ':');
+			if (q==NULL) {
+				fprintf(stderr, "-%c format error\n", op);
+				return -1;
+			}
+			*q = '\0';
+			attrs[i].value = q + 1;
+
+			porcess_next = 0;
+			i++;
+			p = q; /* ++ at end */
+		}
+		if (*p==',') {
+			*p = '\0'; /* ++ at end */
+			porcess_next = 1;
+		}
+	}
+	*pattrs = attrs;
+	*nattrs = n;
+
+	return 0;
+}
+
+static int ibwtest_get_address(const char *address, struct in_addr *addr)
+{
+	if (inet_pton(AF_INET, address, addr) <= 0) {
+		struct hostent *he = gethostbyname(address);
+		if (he == NULL || he->h_length > sizeof(*addr)) {
+			DEBUG(DEBUG_ERR, ("invalid network address '%s'\n", address));
+			return -1;
+		}
+		memcpy(addr, he->h_addr, he->h_length);
+	}
+	return 0;
+}
+
+static int ibwtest_getdests(struct ibwtest_ctx *tcx, char op)
+{
+	int	i;
+	struct ibw_initattr	*attrs = NULL;
+	struct sockaddr_in	*p;
+	char	*tmp;
+
+	tmp = talloc_strdup(tcx, optarg);
+	if (tmp == NULL) return -1;
+	/* hack to reuse the above ibw_initattr parser */
+	if (ibwtest_parse_attrs(tcx, tmp, &attrs, &tcx->naddrs, op))
+		return -1;
+
+	tcx->addrs = talloc_size(tcx,
+		tcx->naddrs * sizeof(struct sockaddr_in));
+	for(i=0; i<tcx->naddrs; i++) {
+		p = tcx->addrs + i;
+		p->sin_family = AF_INET;
+		if (ibwtest_get_address(attrs[i].name, &p->sin_addr))
+			return -1;
+		p->sin_port = htons(atoi(attrs[i].value));
+	}
+
+	return 0;
+}
+
+static int ibwtest_init_server(struct ibwtest_ctx *tcx)
+{
+	if (tcx->naddrs!=1) {
+		fprintf(stderr, "incorrect number of addrs(%d!=1)\n", tcx->naddrs);
+		return -1;
+	}
+
+	if (ibw_bind(tcx->ibwctx, &tcx->addrs[0])) {
+		DEBUG(DEBUG_ERR, ("ERROR: ibw_bind failed\n"));
+		return -1;
+	}
+	
+	if (ibw_listen(tcx->ibwctx, 1)) {
+		DEBUG(DEBUG_ERR, ("ERROR: ibw_listen failed\n"));
+		return -1;
+	}
+
+	/* continued at IBWS_READY */
+	return 0;
+}
+
+static void ibwtest_usage(struct ibwtest_ctx *tcx, char *name)
+{
+	printf("Usage:\n");
+	printf("\t%s -i <id> -o {name:value} -d {addr:port} -t nsec -s\n", name);
+	printf("\t-i <id> is a free text, acting as a server id, max 23 chars [mandatory]\n");
+	printf("\t-o name1:value1,name2:value2,... is a list of (name, value) pairs\n");
+	printf("\t-a addr1:port1,addr2:port2,... is a list of destination ip addresses\n");
+	printf("\t-t nsec delta time between sends in nanosec [default %d]\n", tcx->nsec);
+	printf("\t\t send message periodically and endless when nsec is non-zero\n");
+	printf("\t-s server mode (you have to give exactly one -d address:port in this case)\n");
+	printf("\t-n number of messages to send [default %d]\n", tcx->nmsg);
+	printf("\t-l usec time to sleep in the main loop [default %d]\n", tcx->sleep_usec);
+	printf("\t-v max variable msg size in bytes [default %d], 0=don't send var. size\n", tcx->maxsize);
+	printf("\t-d LogLevel [default %d]\n", DEBUGLEVEL);
+	printf("Press ctrl+C to stop the program.\n");
+}
+
+int main(int argc, char *argv[])
+{
+	int	rc, op;
+	int	result = 1;
+	struct tevent_context *ev = NULL;
+	struct ibwtest_ctx *tcx = NULL;
+	float	usec;
+
+	tcx = talloc_zero(NULL, struct ibwtest_ctx);
+	memset(tcx, 0, sizeof(struct ibwtest_ctx));
+	tcx->nsec = 0;
+	tcx->nmsg = 1000;
+	debuglevel_set(0);
+
+	/* here is the only case we can't avoid using global... */
+	testctx = tcx;
+	signal(SIGINT, ibwtest_sigint_handler);
+	srand((unsigned)time(NULL));
+
+	while ((op=getopt(argc, argv, "i:o:d:m:st:n:l:v:a:")) != -1) {
+		switch (op) {
+		case 'i':
+			tcx->id = talloc_strdup(tcx, optarg);
+			break;
+		case 'o':
+			tcx->opts = talloc_strdup(tcx, optarg);
+			if (tcx->opts) goto cleanup;
+			if (ibwtest_parse_attrs(tcx, tcx->opts, &tcx->attrs,
+				&tcx->nattrs, op))
+				goto cleanup;
+			break;
+		case 'a':
+			if (ibwtest_getdests(tcx, op))
+				goto cleanup;
+			break;
+		case 's':
+			tcx->is_server = 1;
+			break;
+		case 't':
+			tcx->nsec = (unsigned int)atoi(optarg);
+			break;
+		case 'n':
+			tcx->nmsg = atoi(optarg);
+			break;
+		case 'l':
+			tcx->sleep_usec = (unsigned int)atoi(optarg);
+			break;
+		case 'v':
+			tcx->maxsize = (unsigned int)atoi(optarg);
+			break;
+		case 'd':
+			debuglevel_set(atoi(optarg));
+			break;
+		default:
+			fprintf(stderr, "ERROR: unknown option -%c\n", (char)op);
+			ibwtest_usage(tcx, argv[0]);
+			goto cleanup;
+		}
+	}
+	if (tcx->id==NULL) {
+		ibwtest_usage(tcx, argv[0]);
+		goto cleanup;
+	}
+
+	ev = tevent_context_init(NULL);
+	assert(ev);
+
+	tcx->ibwctx = ibw_init(tcx->attrs, tcx->nattrs,
+		tcx,
+		ibwtest_connstate_handler,
+		ibwtest_receive_handler,
+		ev
+	);
+	if (!tcx->ibwctx)
+		goto cleanup;
+
+	if (tcx->is_server)
+		rc = ibwtest_init_server(tcx);
+	else
+		rc = ibwtest_connect_everybody(tcx);
+	if (rc)
+		goto cleanup;
+
+	while(!tcx->kill_me && !tcx->error) {
+		if (tcx->nsec) {
+			tevent_add_timer(ev, tcx,
+					 timeval_current_ofs(0, tcx->nsec),
+					 ibwtest_timeout_handler, tcx);
+		}
+
+		tevent_loop_once(ev);
+
+		if (tcx->sleep_usec)
+			usleep(tcx->sleep_usec);
+	}
+
+	if (!tcx->is_server && tcx->nsent!=0 && !tcx->error) {
+		if (gettimeofday(&tcx->end_time, NULL)) {
+			DEBUG(DEBUG_ERR, ("gettimeofday error %d\n", errno));
+			goto cleanup;
+		}
+		usec = (tcx->end_time.tv_sec - tcx->start_time.tv_sec) * 1000000 +
+				(tcx->end_time.tv_usec - tcx->start_time.tv_usec);
+		printf("usec: %f, nmsg: %d, usec/nmsg: %f\n",
+			usec, tcx->nsent, usec/(float)tcx->nsent);
+	}
+
+	if (!tcx->error)
+		result = 0; /* everything OK */
+
+cleanup:
+	if (tcx)
+		talloc_free(tcx);
+	if (ev)
+		talloc_free(ev);
+	DEBUG(DEBUG_ERR, ("exited with code %d\n", result));
+	return result;
+}