summaryrefslogtreecommitdiffstats
path: root/ctdb/ib/ibw_ctdb_init.c
diff options
context:
space:
mode:
Diffstat (limited to 'ctdb/ib/ibw_ctdb_init.c')
-rw-r--r--ctdb/ib/ibw_ctdb_init.c255
1 files changed, 255 insertions, 0 deletions
diff --git a/ctdb/ib/ibw_ctdb_init.c b/ctdb/ib/ibw_ctdb_init.c
new file mode 100644
index 0000000..f9d00c6
--- /dev/null
+++ b/ctdb/ib/ibw_ctdb_init.c
@@ -0,0 +1,255 @@
+/*
+ * Unix SMB/CIFS implementation.
+ * Join infiniband wrapper and ctdb.
+ *
+ * Copyright (C) Sven Oehme <oehmes@de.ibm.com> 2006
+ *
+ * Major code contributions by Peter Somogyi <psomogyi@gamax.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "replace.h"
+#include "system/network.h"
+
+#include <assert.h>
+#include <talloc.h>
+#include <tevent.h>
+
+#include "lib/util/dlinklist.h"
+#include "lib/util/debug.h"
+
+#include "ctdb_private.h"
+
+#include "common/common.h"
+#include "common/logging.h"
+
+#include "ibwrapper.h"
+#include "ibw_ctdb.h"
+
+static int ctdb_ibw_listen(struct ctdb_context *ctdb, int backlog)
+{
+ struct ibw_ctx *ictx = talloc_get_type(ctdb->transport_data,
+ struct ibw_ctx);
+
+ assert(ictx!=NULL);
+
+ if (ibw_bind(ictx, &ctdb->address->ip)) {
+ DEBUG(DEBUG_CRIT, ("ctdb_ibw_listen: ibw_bind failed\n"));
+ return -1;
+ }
+
+ if (ibw_listen(ictx, backlog)) {
+ DEBUG(DEBUG_CRIT, ("ctdb_ibw_listen: ibw_listen failed\n"));
+ return -1;
+ }
+
+ return 0;
+}
+
+/*
+ * initialise ibw portion of a ctdb node
+ */
+static int ctdb_ibw_add_node(struct ctdb_node *node)
+{
+ struct ibw_ctx *ictx = talloc_get_type(node->ctdb->transport_data,
+ struct ibw_ctx);
+ struct ctdb_ibw_node *cn = talloc_zero(node, struct ctdb_ibw_node);
+
+ assert(cn!=NULL);
+ cn->conn = ibw_conn_new(ictx, node);
+ node->transport_data = (void *)cn;
+
+ return (cn->conn!=NULL ? 0 : -1);
+}
+
+/*
+ * initialise infiniband
+ */
+static int ctdb_ibw_initialise(struct ctdb_context *ctdb)
+{
+ int i, ret;
+
+ ret = ctdb_ibw_init(ctdb);
+ if (ret != 0) {
+ return ret;
+ }
+
+ for (i=0; i<ctdb->num_nodes; i++) {
+ if (ctdb_ibw_add_node(ctdb->nodes[i]) != 0) {
+ DEBUG(DEBUG_CRIT, ("methods->add_node failed at %d\n", i));
+ return -1;
+ }
+ }
+
+ /* listen on our own address */
+ if (ctdb_ibw_listen(ctdb, 10)) /* TODO: backlog as param */
+ return -1;
+
+ return 0;
+}
+
+
+/*
+ * Start infiniband
+ */
+static int ctdb_ibw_start(struct ctdb_context *ctdb)
+{
+ int i;
+
+ /* everything async here */
+ for (i=0;i<ctdb->num_nodes;i++) {
+ struct ctdb_node *node = ctdb->nodes[i];
+ if (!ctdb_same_address(ctdb->address, &node->address)) {
+ ctdb_ibw_node_connect(node);
+ }
+ }
+
+ return 0;
+}
+
+static int ctdb_ibw_send_pkt(struct ibw_conn *conn, uint8_t *data, uint32_t length)
+{
+ void *buf, *key;
+
+ if (ibw_alloc_send_buf(conn, &buf, &key, length)) {
+ DEBUG(DEBUG_ERR, ("queue_pkt/ibw_alloc_send_buf failed\n"));
+ return -1;
+ }
+
+ memcpy(buf, data, length);
+ return ibw_send(conn, buf, key, length);
+}
+
+int ctdb_flush_cn_queue(struct ctdb_ibw_node *cn)
+{
+ struct ctdb_ibw_msg *p;
+ int rc = 0;
+
+ while(cn->queue) {
+ p = cn->queue;
+ rc = ctdb_ibw_send_pkt(cn->conn, p->data, p->length);
+ if (rc)
+ return -1; /* will be retried later when conn is up */
+
+ DLIST_REMOVE(cn->queue, p);
+ cn->qcnt--;
+ talloc_free(p); /* it will talloc_free p->data as well */
+ }
+ assert(cn->qcnt==0);
+ /* cn->queue_last = NULL is not needed - see DLIST_ADD_AFTER */
+
+ return rc;
+}
+
+static int ctdb_ibw_queue_pkt(struct ctdb_node *node, uint8_t *data, uint32_t length)
+{
+ struct ctdb_ibw_node *cn = talloc_get_type(node->transport_data,
+ struct ctdb_ibw_node);
+ int rc;
+
+ assert(length>=sizeof(uint32_t));
+ assert(cn!=NULL);
+
+ if (cn->conn==NULL) {
+ DEBUG(DEBUG_ERR, ("ctdb_ibw_queue_pkt: conn is NULL\n"));
+ return -1;
+ }
+
+ if (cn->conn->state==IBWC_CONNECTED) {
+ rc = ctdb_ibw_send_pkt(cn->conn, data, length);
+ } else {
+ struct ctdb_ibw_msg *p = talloc_zero(cn, struct ctdb_ibw_msg);
+ CTDB_NO_MEMORY(node->ctdb, p);
+
+ p->data = talloc_memdup(p, data, length);
+ CTDB_NO_MEMORY(node->ctdb, p->data);
+
+ p->length = length;
+
+ DLIST_ADD_AFTER(cn->queue, p, cn->queue_last);
+ cn->queue_last = p;
+ cn->qcnt++;
+
+ rc = 0;
+ }
+
+ return rc;
+}
+
+static void ctdb_ibw_restart(struct ctdb_node *node)
+{
+ /* TODO: implement this method for IB */
+ DEBUG(DEBUG_ALERT,("WARNING: method restart is not yet implemented for IB\n"));
+}
+
+/*
+ * transport packet allocator - allows transport to control memory for packets
+ */
+static void *ctdb_ibw_allocate_pkt(TALLOC_CTX *mem_ctx, size_t size)
+{
+ /* TODO: use ibw_alloc_send_buf instead... */
+ return talloc_size(mem_ctx, size);
+}
+
+#ifdef __NOTDEF__
+
+static int ctdb_ibw_stop(struct ctdb_context *cctx)
+{
+ struct ibw_ctx *ictx = talloc_get_type(cctx->private_data, struct ibw_ctx);
+
+ assert(ictx!=NULL);
+ return ibw_stop(ictx);
+}
+
+#endif /* __NOTDEF__ */
+
+static const struct ctdb_methods ctdb_ibw_methods = {
+ .initialise= ctdb_ibw_initialise,
+ .start = ctdb_ibw_start,
+ .queue_pkt = ctdb_ibw_queue_pkt,
+ .add_node = ctdb_ibw_add_node,
+ .allocate_pkt = ctdb_ibw_allocate_pkt,
+ .restart = ctdb_ibw_restart,
+
+// .stop = ctdb_ibw_stop
+};
+
+/*
+ * initialise ibw portion of ctdb
+ */
+int ctdb_ibw_init(struct ctdb_context *ctdb)
+{
+ struct ibw_ctx *ictx;
+
+ DEBUG(DEBUG_DEBUG, ("ctdb_ibw_init invoked...\n"));
+ ictx = ibw_init(
+ NULL, //struct ibw_initattr *attr, /* TODO */
+ 0, //int nattr, /* TODO */
+ ctdb,
+ ctdb_ibw_connstate_handler,
+ ctdb_ibw_receive_handler,
+ ctdb->ev);
+
+ if (ictx==NULL) {
+ DEBUG(DEBUG_CRIT, ("ctdb_ibw_init: ibw_init failed\n"));
+ return -1;
+ }
+
+ ctdb->methods = &ctdb_ibw_methods;
+ ctdb->transport_data = ictx;
+
+ DEBUG(DEBUG_DEBUG, ("ctdb_ibw_init succeeded.\n"));
+ return 0;
+}