Adding upstream version 4.19.249.upstream/4.19.249upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 2428 insertions, 0 deletions

diff --git a/arch/sparc/kernel/ldc.c b/arch/sparc/kernel/ldc.c
new file mode 100644
index 000000000..c0fa3ef6c
--- /dev/null
+++ b/arch/sparc/kernel/ldc.c
@@ -0,0 +1,2428 @@
+// SPDX-License-Identifier: GPL-2.0
+/* ldc.c: Logical Domain Channel link-layer protocol driver.
+ *
+ * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net>
+ */
+
+#include <linux/kernel.h>
+#include <linux/export.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/delay.h>
+#include <linux/errno.h>
+#include <linux/string.h>
+#include <linux/scatterlist.h>
+#include <linux/interrupt.h>
+#include <linux/list.h>
+#include <linux/init.h>
+#include <linux/bitmap.h>
+#include <asm/iommu-common.h>
+
+#include <asm/hypervisor.h>
+#include <asm/iommu.h>
+#include <asm/page.h>
+#include <asm/ldc.h>
+#include <asm/mdesc.h>
+
+#define DRV_MODULE_NAME		"ldc"
+#define PFX DRV_MODULE_NAME	": "
+#define DRV_MODULE_VERSION	"1.1"
+#define DRV_MODULE_RELDATE	"July 22, 2008"
+
+#define COOKIE_PGSZ_CODE	0xf000000000000000ULL
+#define COOKIE_PGSZ_CODE_SHIFT	60ULL
+
+
+static char version[] =
+	DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
+
+/* Packet header layout for unreliable and reliable mode frames.
+ * When in RAW mode, packets are simply straight 64-byte payloads
+ * with no headers.
+ */
+struct ldc_packet {
+	u8			type;
+#define LDC_CTRL		0x01
+#define LDC_DATA		0x02
+#define LDC_ERR			0x10
+
+	u8			stype;
+#define LDC_INFO		0x01
+#define LDC_ACK			0x02
+#define LDC_NACK		0x04
+
+	u8			ctrl;
+#define LDC_VERS		0x01 /* Link Version		*/
+#define LDC_RTS			0x02 /* Request To Send		*/
+#define LDC_RTR			0x03 /* Ready To Receive	*/
+#define LDC_RDX			0x04 /* Ready for Data eXchange	*/
+#define LDC_CTRL_MSK		0x0f
+
+	u8			env;
+#define LDC_LEN			0x3f
+#define LDC_FRAG_MASK		0xc0
+#define LDC_START		0x40
+#define LDC_STOP		0x80
+
+	u32			seqid;
+
+	union {
+		u8		u_data[LDC_PACKET_SIZE - 8];
+		struct {
+			u32	pad;
+			u32	ackid;
+			u8	r_data[LDC_PACKET_SIZE - 8 - 8];
+		} r;
+	} u;
+};
+
+struct ldc_version {
+	u16 major;
+	u16 minor;
+};
+
+/* Ordered from largest major to lowest.  */
+static struct ldc_version ver_arr[] = {
+	{ .major = 1, .minor = 0 },
+};
+
+#define LDC_DEFAULT_MTU			(4 * LDC_PACKET_SIZE)
+#define LDC_DEFAULT_NUM_ENTRIES		(PAGE_SIZE / LDC_PACKET_SIZE)
+
+struct ldc_channel;
+
+struct ldc_mode_ops {
+	int (*write)(struct ldc_channel *, const void *, unsigned int);
+	int (*read)(struct ldc_channel *, void *, unsigned int);
+};
+
+static const struct ldc_mode_ops raw_ops;
+static const struct ldc_mode_ops nonraw_ops;
+static const struct ldc_mode_ops stream_ops;
+
+int ldom_domaining_enabled;
+
+struct ldc_iommu {
+	/* Protects ldc_unmap.  */
+	spinlock_t			lock;
+	struct ldc_mtable_entry		*page_table;
+	struct iommu_map_table		iommu_map_table;
+};
+
+struct ldc_channel {
+	/* Protects all operations that depend upon channel state.  */
+	spinlock_t			lock;
+
+	unsigned long			id;
+
+	u8				*mssbuf;
+	u32				mssbuf_len;
+	u32				mssbuf_off;
+
+	struct ldc_packet		*tx_base;
+	unsigned long			tx_head;
+	unsigned long			tx_tail;
+	unsigned long			tx_num_entries;
+	unsigned long			tx_ra;
+
+	unsigned long			tx_acked;
+
+	struct ldc_packet		*rx_base;
+	unsigned long			rx_head;
+	unsigned long			rx_tail;
+	unsigned long			rx_num_entries;
+	unsigned long			rx_ra;
+
+	u32				rcv_nxt;
+	u32				snd_nxt;
+
+	unsigned long			chan_state;
+
+	struct ldc_channel_config	cfg;
+	void				*event_arg;
+
+	const struct ldc_mode_ops	*mops;
+
+	struct ldc_iommu		iommu;
+
+	struct ldc_version		ver;
+
+	u8				hs_state;
+#define LDC_HS_CLOSED			0x00
+#define LDC_HS_OPEN			0x01
+#define LDC_HS_GOTVERS			0x02
+#define LDC_HS_SENTRTR			0x03
+#define LDC_HS_GOTRTR			0x04
+#define LDC_HS_COMPLETE			0x10
+
+	u8				flags;
+#define LDC_FLAG_ALLOCED_QUEUES		0x01
+#define LDC_FLAG_REGISTERED_QUEUES	0x02
+#define LDC_FLAG_REGISTERED_IRQS	0x04
+#define LDC_FLAG_RESET			0x10
+
+	u8				mss;
+	u8				state;
+
+#define LDC_IRQ_NAME_MAX		32
+	char				rx_irq_name[LDC_IRQ_NAME_MAX];
+	char				tx_irq_name[LDC_IRQ_NAME_MAX];
+
+	struct hlist_head		mh_list;
+
+	struct hlist_node		list;
+};
+
+#define ldcdbg(TYPE, f, a...) \
+do {	if (lp->cfg.debug & LDC_DEBUG_##TYPE) \
+		printk(KERN_INFO PFX "ID[%lu] " f, lp->id, ## a); \
+} while (0)
+
+#define	LDC_ABORT(lp)	ldc_abort((lp), __func__)
+
+static const char *state_to_str(u8 state)
+{
+	switch (state) {
+	case LDC_STATE_INVALID:
+		return "INVALID";
+	case LDC_STATE_INIT:
+		return "INIT";
+	case LDC_STATE_BOUND:
+		return "BOUND";
+	case LDC_STATE_READY:
+		return "READY";
+	case LDC_STATE_CONNECTED:
+		return "CONNECTED";
+	default:
+		return "<UNKNOWN>";
+	}
+}
+
+static unsigned long __advance(unsigned long off, unsigned long num_entries)
+{
+	off += LDC_PACKET_SIZE;
+	if (off == (num_entries * LDC_PACKET_SIZE))
+		off = 0;
+
+	return off;
+}
+
+static unsigned long rx_advance(struct ldc_channel *lp, unsigned long off)
+{
+	return __advance(off, lp->rx_num_entries);
+}
+
+static unsigned long tx_advance(struct ldc_channel *lp, unsigned long off)
+{
+	return __advance(off, lp->tx_num_entries);
+}
+
+static struct ldc_packet *handshake_get_tx_packet(struct ldc_channel *lp,
+						  unsigned long *new_tail)
+{
+	struct ldc_packet *p;
+	unsigned long t;
+
+	t = tx_advance(lp, lp->tx_tail);
+	if (t == lp->tx_head)
+		return NULL;
+
+	*new_tail = t;
+
+	p = lp->tx_base;
+	return p + (lp->tx_tail / LDC_PACKET_SIZE);
+}
+
+/* When we are in reliable or stream mode, have to track the next packet
+ * we haven't gotten an ACK for in the TX queue using tx_acked.  We have
+ * to be careful not to stomp over the queue past that point.  During
+ * the handshake, we don't have TX data packets pending in the queue
+ * and that's why handshake_get_tx_packet() need not be mindful of
+ * lp->tx_acked.
+ */
+static unsigned long head_for_data(struct ldc_channel *lp)
+{
+	if (lp->cfg.mode == LDC_MODE_STREAM)
+		return lp->tx_acked;
+	return lp->tx_head;
+}
+
+static int tx_has_space_for(struct ldc_channel *lp, unsigned int size)
+{
+	unsigned long limit, tail, new_tail, diff;
+	unsigned int mss;
+
+	limit = head_for_data(lp);
+	tail = lp->tx_tail;
+	new_tail = tx_advance(lp, tail);
+	if (new_tail == limit)
+		return 0;
+
+	if (limit > new_tail)
+		diff = limit - new_tail;
+	else
+		diff = (limit +
+			((lp->tx_num_entries * LDC_PACKET_SIZE) - new_tail));
+	diff /= LDC_PACKET_SIZE;
+	mss = lp->mss;
+
+	if (diff * mss < size)
+		return 0;
+
+	return 1;
+}
+
+static struct ldc_packet *data_get_tx_packet(struct ldc_channel *lp,
+					     unsigned long *new_tail)
+{
+	struct ldc_packet *p;
+	unsigned long h, t;
+
+	h = head_for_data(lp);
+	t = tx_advance(lp, lp->tx_tail);
+	if (t == h)
+		return NULL;
+
+	*new_tail = t;
+
+	p = lp->tx_base;
+	return p + (lp->tx_tail / LDC_PACKET_SIZE);
+}
+
+static int set_tx_tail(struct ldc_channel *lp, unsigned long tail)
+{
+	unsigned long orig_tail = lp->tx_tail;
+	int limit = 1000;
+
+	lp->tx_tail = tail;
+	while (limit-- > 0) {
+		unsigned long err;
+
+		err = sun4v_ldc_tx_set_qtail(lp->id, tail);
+		if (!err)
+			return 0;
+
+		if (err != HV_EWOULDBLOCK) {
+			lp->tx_tail = orig_tail;
+			return -EINVAL;
+		}
+		udelay(1);
+	}
+
+	lp->tx_tail = orig_tail;
+	return -EBUSY;
+}
+
+/* This just updates the head value in the hypervisor using
+ * a polling loop with a timeout.  The caller takes care of
+ * upating software state representing the head change, if any.
+ */
+static int __set_rx_head(struct ldc_channel *lp, unsigned long head)
+{
+	int limit = 1000;
+
+	while (limit-- > 0) {
+		unsigned long err;
+
+		err = sun4v_ldc_rx_set_qhead(lp->id, head);
+		if (!err)
+			return 0;
+
+		if (err != HV_EWOULDBLOCK)
+			return -EINVAL;
+
+		udelay(1);
+	}
+
+	return -EBUSY;
+}
+
+static int send_tx_packet(struct ldc_channel *lp,
+			  struct ldc_packet *p,
+			  unsigned long new_tail)
+{
+	BUG_ON(p != (lp->tx_base + (lp->tx_tail / LDC_PACKET_SIZE)));
+
+	return set_tx_tail(lp, new_tail);
+}
+
+static struct ldc_packet *handshake_compose_ctrl(struct ldc_channel *lp,
+						 u8 stype, u8 ctrl,
+						 void *data, int dlen,
+						 unsigned long *new_tail)
+{
+	struct ldc_packet *p = handshake_get_tx_packet(lp, new_tail);
+
+	if (p) {
+		memset(p, 0, sizeof(*p));
+		p->type = LDC_CTRL;
+		p->stype = stype;
+		p->ctrl = ctrl;
+		if (data)
+			memcpy(p->u.u_data, data, dlen);
+	}
+	return p;
+}
+
+static int start_handshake(struct ldc_channel *lp)
+{
+	struct ldc_packet *p;
+	struct ldc_version *ver;
+	unsigned long new_tail;
+
+	ver = &ver_arr[0];
+
+	ldcdbg(HS, "SEND VER INFO maj[%u] min[%u]\n",
+	       ver->major, ver->minor);
+
+	p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS,
+				   ver, sizeof(*ver), &new_tail);
+	if (p) {
+		int err = send_tx_packet(lp, p, new_tail);
+		if (!err)
+			lp->flags &= ~LDC_FLAG_RESET;
+		return err;
+	}
+	return -EBUSY;
+}
+
+static int send_version_nack(struct ldc_channel *lp,
+			     u16 major, u16 minor)
+{
+	struct ldc_packet *p;
+	struct ldc_version ver;
+	unsigned long new_tail;
+
+	ver.major = major;
+	ver.minor = minor;
+
+	p = handshake_compose_ctrl(lp, LDC_NACK, LDC_VERS,
+				   &ver, sizeof(ver), &new_tail);
+	if (p) {
+		ldcdbg(HS, "SEND VER NACK maj[%u] min[%u]\n",
+		       ver.major, ver.minor);
+
+		return send_tx_packet(lp, p, new_tail);
+	}
+	return -EBUSY;
+}
+
+static int send_version_ack(struct ldc_channel *lp,
+			    struct ldc_version *vp)
+{
+	struct ldc_packet *p;
+	unsigned long new_tail;
+
+	p = handshake_compose_ctrl(lp, LDC_ACK, LDC_VERS,
+				   vp, sizeof(*vp), &new_tail);
+	if (p) {
+		ldcdbg(HS, "SEND VER ACK maj[%u] min[%u]\n",
+		       vp->major, vp->minor);
+
+		return send_tx_packet(lp, p, new_tail);
+	}
+	return -EBUSY;
+}
+
+static int send_rts(struct ldc_channel *lp)
+{
+	struct ldc_packet *p;
+	unsigned long new_tail;
+
+	p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTS, NULL, 0,
+				   &new_tail);
+	if (p) {
+		p->env = lp->cfg.mode;
+		p->seqid = 0;
+		lp->rcv_nxt = 0;
+
+		ldcdbg(HS, "SEND RTS env[0x%x] seqid[0x%x]\n",
+		       p->env, p->seqid);
+
+		return send_tx_packet(lp, p, new_tail);
+	}
+	return -EBUSY;
+}
+
+static int send_rtr(struct ldc_channel *lp)
+{
+	struct ldc_packet *p;
+	unsigned long new_tail;
+
+	p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTR, NULL, 0,
+				   &new_tail);
+	if (p) {
+		p->env = lp->cfg.mode;
+		p->seqid = 0;
+
+		ldcdbg(HS, "SEND RTR env[0x%x] seqid[0x%x]\n",
+		       p->env, p->seqid);
+
+		return send_tx_packet(lp, p, new_tail);
+	}
+	return -EBUSY;
+}
+
+static int send_rdx(struct ldc_channel *lp)
+{
+	struct ldc_packet *p;
+	unsigned long new_tail;
+
+	p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RDX, NULL, 0,
+				   &new_tail);
+	if (p) {
+		p->env = 0;
+		p->seqid = ++lp->snd_nxt;
+		p->u.r.ackid = lp->rcv_nxt;
+
+		ldcdbg(HS, "SEND RDX env[0x%x] seqid[0x%x] ackid[0x%x]\n",
+		       p->env, p->seqid, p->u.r.ackid);
+
+		return send_tx_packet(lp, p, new_tail);
+	}
+	return -EBUSY;
+}
+
+static int send_data_nack(struct ldc_channel *lp, struct ldc_packet *data_pkt)
+{
+	struct ldc_packet *p;
+	unsigned long new_tail;
+	int err;
+
+	p = data_get_tx_packet(lp, &new_tail);
+	if (!p)
+		return -EBUSY;
+	memset(p, 0, sizeof(*p));
+	p->type = data_pkt->type;
+	p->stype = LDC_NACK;
+	p->ctrl = data_pkt->ctrl & LDC_CTRL_MSK;
+	p->seqid = lp->snd_nxt + 1;
+	p->u.r.ackid = lp->rcv_nxt;
+
+	ldcdbg(HS, "SEND DATA NACK type[0x%x] ctl[0x%x] seq[0x%x] ack[0x%x]\n",
+	       p->type, p->ctrl, p->seqid, p->u.r.ackid);
+
+	err = send_tx_packet(lp, p, new_tail);
+	if (!err)
+		lp->snd_nxt++;
+
+	return err;
+}
+
+static int ldc_abort(struct ldc_channel *lp, const char *msg)
+{
+	unsigned long hv_err;
+
+	ldcdbg(STATE, "ABORT[%s]\n", msg);
+	ldc_print(lp);
+
+	/* We report but do not act upon the hypervisor errors because
+	 * there really isn't much we can do if they fail at this point.
+	 */
+	hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
+	if (hv_err)
+		printk(KERN_ERR PFX "ldc_abort: "
+		       "sun4v_ldc_tx_qconf(%lx,%lx,%lx) failed, err=%lu\n",
+		       lp->id, lp->tx_ra, lp->tx_num_entries, hv_err);
+
+	hv_err = sun4v_ldc_tx_get_state(lp->id,
+					&lp->tx_head,
+					&lp->tx_tail,
+					&lp->chan_state);
+	if (hv_err)
+		printk(KERN_ERR PFX "ldc_abort: "
+		       "sun4v_ldc_tx_get_state(%lx,...) failed, err=%lu\n",
+		       lp->id, hv_err);
+
+	hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
+	if (hv_err)
+		printk(KERN_ERR PFX "ldc_abort: "
+		       "sun4v_ldc_rx_qconf(%lx,%lx,%lx) failed, err=%lu\n",
+		       lp->id, lp->rx_ra, lp->rx_num_entries, hv_err);
+
+	/* Refetch the RX queue state as well, because we could be invoked
+	 * here in the queue processing context.
+	 */
+	hv_err = sun4v_ldc_rx_get_state(lp->id,
+					&lp->rx_head,
+					&lp->rx_tail,
+					&lp->chan_state);
+	if (hv_err)
+		printk(KERN_ERR PFX "ldc_abort: "
+		       "sun4v_ldc_rx_get_state(%lx,...) failed, err=%lu\n",
+		       lp->id, hv_err);
+
+	return -ECONNRESET;
+}
+
+static struct ldc_version *find_by_major(u16 major)
+{
+	struct ldc_version *ret = NULL;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(ver_arr); i++) {
+		struct ldc_version *v = &ver_arr[i];
+		if (v->major <= major) {
+			ret = v;
+			break;
+		}
+	}
+	return ret;
+}
+
+static int process_ver_info(struct ldc_channel *lp, struct ldc_version *vp)
+{
+	struct ldc_version *vap;
+	int err;
+
+	ldcdbg(HS, "GOT VERSION INFO major[%x] minor[%x]\n",
+	       vp->major, vp->minor);
+
+	if (lp->hs_state == LDC_HS_GOTVERS) {
+		lp->hs_state = LDC_HS_OPEN;
+		memset(&lp->ver, 0, sizeof(lp->ver));
+	}
+
+	vap = find_by_major(vp->major);
+	if (!vap) {
+		err = send_version_nack(lp, 0, 0);
+	} else if (vap->major != vp->major) {
+		err = send_version_nack(lp, vap->major, vap->minor);
+	} else {
+		struct ldc_version ver = *vp;
+		if (ver.minor > vap->minor)
+			ver.minor = vap->minor;
+		err = send_version_ack(lp, &ver);
+		if (!err) {
+			lp->ver = ver;
+			lp->hs_state = LDC_HS_GOTVERS;
+		}
+	}
+	if (err)
+		return LDC_ABORT(lp);
+
+	return 0;
+}
+
+static int process_ver_ack(struct ldc_channel *lp, struct ldc_version *vp)
+{
+	ldcdbg(HS, "GOT VERSION ACK major[%x] minor[%x]\n",
+	       vp->major, vp->minor);
+
+	if (lp->hs_state == LDC_HS_GOTVERS) {
+		if (lp->ver.major != vp->major ||
+		    lp->ver.minor != vp->minor)
+			return LDC_ABORT(lp);
+	} else {
+		lp->ver = *vp;
+		lp->hs_state = LDC_HS_GOTVERS;
+	}
+	if (send_rts(lp))
+		return LDC_ABORT(lp);
+	return 0;
+}
+
+static int process_ver_nack(struct ldc_channel *lp, struct ldc_version *vp)
+{
+	struct ldc_version *vap;
+	struct ldc_packet *p;
+	unsigned long new_tail;
+
+	if (vp->major == 0 && vp->minor == 0)
+		return LDC_ABORT(lp);
+
+	vap = find_by_major(vp->major);
+	if (!vap)
+		return LDC_ABORT(lp);
+
+	p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS,
+					   vap, sizeof(*vap),
+					   &new_tail);
+	if (!p)
+		return LDC_ABORT(lp);
+
+	return send_tx_packet(lp, p, new_tail);
+}
+
+static int process_version(struct ldc_channel *lp,
+			   struct ldc_packet *p)
+{
+	struct ldc_version *vp;
+
+	vp = (struct ldc_version *) p->u.u_data;
+
+	switch (p->stype) {
+	case LDC_INFO:
+		return process_ver_info(lp, vp);
+
+	case LDC_ACK:
+		return process_ver_ack(lp, vp);
+
+	case LDC_NACK:
+		return process_ver_nack(lp, vp);
+
+	default:
+		return LDC_ABORT(lp);
+	}
+}
+
+static int process_rts(struct ldc_channel *lp,
+		       struct ldc_packet *p)
+{
+	ldcdbg(HS, "GOT RTS stype[%x] seqid[%x] env[%x]\n",
+	       p->stype, p->seqid, p->env);
+
+	if (p->stype     != LDC_INFO	   ||
+	    lp->hs_state != LDC_HS_GOTVERS ||
+	    p->env       != lp->cfg.mode)
+		return LDC_ABORT(lp);
+
+	lp->snd_nxt = p->seqid;
+	lp->rcv_nxt = p->seqid;
+	lp->hs_state = LDC_HS_SENTRTR;
+	if (send_rtr(lp))
+		return LDC_ABORT(lp);
+
+	return 0;
+}
+
+static int process_rtr(struct ldc_channel *lp,
+		       struct ldc_packet *p)
+{
+	ldcdbg(HS, "GOT RTR stype[%x] seqid[%x] env[%x]\n",
+	       p->stype, p->seqid, p->env);
+
+	if (p->stype     != LDC_INFO ||
+	    p->env       != lp->cfg.mode)
+		return LDC_ABORT(lp);
+
+	lp->snd_nxt = p->seqid;
+	lp->hs_state = LDC_HS_COMPLETE;
+	ldc_set_state(lp, LDC_STATE_CONNECTED);
+	send_rdx(lp);
+
+	return LDC_EVENT_UP;
+}
+
+static int rx_seq_ok(struct ldc_channel *lp, u32 seqid)
+{
+	return lp->rcv_nxt + 1 == seqid;
+}
+
+static int process_rdx(struct ldc_channel *lp,
+		       struct ldc_packet *p)
+{
+	ldcdbg(HS, "GOT RDX stype[%x] seqid[%x] env[%x] ackid[%x]\n",
+	       p->stype, p->seqid, p->env, p->u.r.ackid);
+
+	if (p->stype != LDC_INFO ||
+	    !(rx_seq_ok(lp, p->seqid)))
+		return LDC_ABORT(lp);
+
+	lp->rcv_nxt = p->seqid;
+
+	lp->hs_state = LDC_HS_COMPLETE;
+	ldc_set_state(lp, LDC_STATE_CONNECTED);
+
+	return LDC_EVENT_UP;
+}
+
+static int process_control_frame(struct ldc_channel *lp,
+				 struct ldc_packet *p)
+{
+	switch (p->ctrl) {
+	case LDC_VERS:
+		return process_version(lp, p);
+
+	case LDC_RTS:
+		return process_rts(lp, p);
+
+	case LDC_RTR:
+		return process_rtr(lp, p);
+
+	case LDC_RDX:
+		return process_rdx(lp, p);
+
+	default:
+		return LDC_ABORT(lp);
+	}
+}
+
+static int process_error_frame(struct ldc_channel *lp,
+			       struct ldc_packet *p)
+{
+	return LDC_ABORT(lp);
+}
+
+static int process_data_ack(struct ldc_channel *lp,
+			    struct ldc_packet *ack)
+{
+	unsigned long head = lp->tx_acked;
+	u32 ackid = ack->u.r.ackid;
+
+	while (1) {
+		struct ldc_packet *p = lp->tx_base + (head / LDC_PACKET_SIZE);
+
+		head = tx_advance(lp, head);
+
+		if (p->seqid == ackid) {
+			lp->tx_acked = head;
+			return 0;
+		}
+		if (head == lp->tx_tail)
+			return LDC_ABORT(lp);
+	}
+
+	return 0;
+}
+
+static void send_events(struct ldc_channel *lp, unsigned int event_mask)
+{
+	if (event_mask & LDC_EVENT_RESET)
+		lp->cfg.event(lp->event_arg, LDC_EVENT_RESET);
+	if (event_mask & LDC_EVENT_UP)
+		lp->cfg.event(lp->event_arg, LDC_EVENT_UP);
+	if (event_mask & LDC_EVENT_DATA_READY)
+		lp->cfg.event(lp->event_arg, LDC_EVENT_DATA_READY);
+}
+
+static irqreturn_t ldc_rx(int irq, void *dev_id)
+{
+	struct ldc_channel *lp = dev_id;
+	unsigned long orig_state, flags;
+	unsigned int event_mask;
+
+	spin_lock_irqsave(&lp->lock, flags);
+
+	orig_state = lp->chan_state;
+
+	/* We should probably check for hypervisor errors here and
+	 * reset the LDC channel if we get one.
+	 */
+	sun4v_ldc_rx_get_state(lp->id,
+			       &lp->rx_head,
+			       &lp->rx_tail,
+			       &lp->chan_state);
+
+	ldcdbg(RX, "RX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n",
+	       orig_state, lp->chan_state, lp->rx_head, lp->rx_tail);
+
+	event_mask = 0;
+
+	if (lp->cfg.mode == LDC_MODE_RAW &&
+	    lp->chan_state == LDC_CHANNEL_UP) {
+		lp->hs_state = LDC_HS_COMPLETE;
+		ldc_set_state(lp, LDC_STATE_CONNECTED);
+
+		/*
+		 * Generate an LDC_EVENT_UP event if the channel
+		 * was not already up.
+		 */
+		if (orig_state != LDC_CHANNEL_UP) {
+			event_mask |= LDC_EVENT_UP;
+			orig_state = lp->chan_state;
+		}
+	}
+
+	/* If we are in reset state, flush the RX queue and ignore
+	 * everything.
+	 */
+	if (lp->flags & LDC_FLAG_RESET) {
+		(void) ldc_rx_reset(lp);
+		goto out;
+	}
+
+	/* Once we finish the handshake, we let the ldc_read()
+	 * paths do all of the control frame and state management.
+	 * Just trigger the callback.
+	 */
+	if (lp->hs_state == LDC_HS_COMPLETE) {
+handshake_complete:
+		if (lp->chan_state != orig_state) {
+			unsigned int event = LDC_EVENT_RESET;
+
+			if (lp->chan_state == LDC_CHANNEL_UP)
+				event = LDC_EVENT_UP;
+
+			event_mask |= event;
+		}
+		if (lp->rx_head != lp->rx_tail)
+			event_mask |= LDC_EVENT_DATA_READY;
+
+		goto out;
+	}
+
+	if (lp->chan_state != orig_state)
+		goto out;
+
+	while (lp->rx_head != lp->rx_tail) {
+		struct ldc_packet *p;
+		unsigned long new;
+		int err;
+
+		p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE);
+
+		switch (p->type) {
+		case LDC_CTRL:
+			err = process_control_frame(lp, p);
+			if (err > 0)
+				event_mask |= err;
+			break;
+
+		case LDC_DATA:
+			event_mask |= LDC_EVENT_DATA_READY;
+			err = 0;
+			break;
+
+		case LDC_ERR:
+			err = process_error_frame(lp, p);
+			break;
+
+		default:
+			err = LDC_ABORT(lp);
+			break;
+		}
+
+		if (err < 0)
+			break;
+
+		new = lp->rx_head;
+		new += LDC_PACKET_SIZE;
+		if (new == (lp->rx_num_entries * LDC_PACKET_SIZE))
+			new = 0;
+		lp->rx_head = new;
+
+		err = __set_rx_head(lp, new);
+		if (err < 0) {
+			(void) LDC_ABORT(lp);
+			break;
+		}
+		if (lp->hs_state == LDC_HS_COMPLETE)
+			goto handshake_complete;
+	}
+
+out:
+	spin_unlock_irqrestore(&lp->lock, flags);
+
+	send_events(lp, event_mask);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t ldc_tx(int irq, void *dev_id)
+{
+	struct ldc_channel *lp = dev_id;
+	unsigned long flags, orig_state;
+	unsigned int event_mask = 0;
+
+	spin_lock_irqsave(&lp->lock, flags);
+
+	orig_state = lp->chan_state;
+
+	/* We should probably check for hypervisor errors here and
+	 * reset the LDC channel if we get one.
+	 */
+	sun4v_ldc_tx_get_state(lp->id,
+			       &lp->tx_head,
+			       &lp->tx_tail,
+			       &lp->chan_state);
+
+	ldcdbg(TX, " TX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n",
+	       orig_state, lp->chan_state, lp->tx_head, lp->tx_tail);
+
+	if (lp->cfg.mode == LDC_MODE_RAW &&
+	    lp->chan_state == LDC_CHANNEL_UP) {
+		lp->hs_state = LDC_HS_COMPLETE;
+		ldc_set_state(lp, LDC_STATE_CONNECTED);
+
+		/*
+		 * Generate an LDC_EVENT_UP event if the channel
+		 * was not already up.
+		 */
+		if (orig_state != LDC_CHANNEL_UP) {
+			event_mask |= LDC_EVENT_UP;
+			orig_state = lp->chan_state;
+		}
+	}
+
+	spin_unlock_irqrestore(&lp->lock, flags);
+
+	send_events(lp, event_mask);
+
+	return IRQ_HANDLED;
+}
+
+/* XXX ldc_alloc() and ldc_free() needs to run under a mutex so
+ * XXX that addition and removal from the ldc_channel_list has
+ * XXX atomicity, otherwise the __ldc_channel_exists() check is
+ * XXX totally pointless as another thread can slip into ldc_alloc()
+ * XXX and add a channel with the same ID.  There also needs to be
+ * XXX a spinlock for ldc_channel_list.
+ */
+static HLIST_HEAD(ldc_channel_list);
+
+static int __ldc_channel_exists(unsigned long id)
+{
+	struct ldc_channel *lp;
+
+	hlist_for_each_entry(lp, &ldc_channel_list, list) {
+		if (lp->id == id)
+			return 1;
+	}
+	return 0;
+}
+
+static int alloc_queue(const char *name, unsigned long num_entries,
+		       struct ldc_packet **base, unsigned long *ra)
+{
+	unsigned long size, order;
+	void *q;
+
+	size = num_entries * LDC_PACKET_SIZE;
+	order = get_order(size);
+
+	q = (void *) __get_free_pages(GFP_KERNEL, order);
+	if (!q) {
+		printk(KERN_ERR PFX "Alloc of %s queue failed with "
+		       "size=%lu order=%lu\n", name, size, order);
+		return -ENOMEM;
+	}
+
+	memset(q, 0, PAGE_SIZE << order);
+
+	*base = q;
+	*ra = __pa(q);
+
+	return 0;
+}
+
+static void free_queue(unsigned long num_entries, struct ldc_packet *q)
+{
+	unsigned long size, order;
+
+	if (!q)
+		return;
+
+	size = num_entries * LDC_PACKET_SIZE;
+	order = get_order(size);
+
+	free_pages((unsigned long)q, order);
+}
+
+static unsigned long ldc_cookie_to_index(u64 cookie, void *arg)
+{
+	u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT;
+	/* struct ldc_iommu *ldc_iommu = (struct ldc_iommu *)arg; */
+
+	cookie &= ~COOKIE_PGSZ_CODE;
+
+	return (cookie >> (13ULL + (szcode * 3ULL)));
+}
+
+static void ldc_demap(struct ldc_iommu *iommu, unsigned long id, u64 cookie,
+		      unsigned long entry, unsigned long npages)
+{
+	struct ldc_mtable_entry *base;
+	unsigned long i, shift;
+
+	shift = (cookie >> COOKIE_PGSZ_CODE_SHIFT) * 3;
+	base = iommu->page_table + entry;
+	for (i = 0; i < npages; i++) {
+		if (base->cookie)
+			sun4v_ldc_revoke(id, cookie + (i << shift),
+					 base->cookie);
+		base->mte = 0;
+	}
+}
+
+/* XXX Make this configurable... XXX */
+#define LDC_IOTABLE_SIZE	(8 * 1024)
+
+static int ldc_iommu_init(const char *name, struct ldc_channel *lp)
+{
+	unsigned long sz, num_tsb_entries, tsbsize, order;
+	struct ldc_iommu *ldc_iommu = &lp->iommu;
+	struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;
+	struct ldc_mtable_entry *table;
+	unsigned long hv_err;
+	int err;
+
+	num_tsb_entries = LDC_IOTABLE_SIZE;
+	tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
+	spin_lock_init(&ldc_iommu->lock);
+
+	sz = num_tsb_entries / 8;
+	sz = (sz + 7UL) & ~7UL;
+	iommu->map = kzalloc(sz, GFP_KERNEL);
+	if (!iommu->map) {
+		printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz);
+		return -ENOMEM;
+	}
+	iommu_tbl_pool_init(iommu, num_tsb_entries, PAGE_SHIFT,
+			    NULL, false /* no large pool */,
+			    1 /* npools */,
+			    true /* skip span boundary check */);
+
+	order = get_order(tsbsize);
+
+	table = (struct ldc_mtable_entry *)
+		__get_free_pages(GFP_KERNEL, order);
+	err = -ENOMEM;
+	if (!table) {
+		printk(KERN_ERR PFX "Alloc of MTE table failed, "
+		       "size=%lu order=%lu\n", tsbsize, order);
+		goto out_free_map;
+	}
+
+	memset(table, 0, PAGE_SIZE << order);
+
+	ldc_iommu->page_table = table;
+
+	hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table),
+					 num_tsb_entries);
+	err = -EINVAL;
+	if (hv_err)
+		goto out_free_table;
+
+	return 0;
+
+out_free_table:
+	free_pages((unsigned long) table, order);
+	ldc_iommu->page_table = NULL;
+
+out_free_map:
+	kfree(iommu->map);
+	iommu->map = NULL;
+
+	return err;
+}
+
+static void ldc_iommu_release(struct ldc_channel *lp)
+{
+	struct ldc_iommu *ldc_iommu = &lp->iommu;
+	struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;
+	unsigned long num_tsb_entries, tsbsize, order;
+
+	(void) sun4v_ldc_set_map_table(lp->id, 0, 0);
+
+	num_tsb_entries = iommu->poolsize * iommu->nr_pools;
+	tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
+	order = get_order(tsbsize);
+
+	free_pages((unsigned long) ldc_iommu->page_table, order);
+	ldc_iommu->page_table = NULL;
+
+	kfree(iommu->map);
+	iommu->map = NULL;
+}
+
+struct ldc_channel *ldc_alloc(unsigned long id,
+			      const struct ldc_channel_config *cfgp,
+			      void *event_arg,
+			      const char *name)
+{
+	struct ldc_channel *lp;
+	const struct ldc_mode_ops *mops;
+	unsigned long dummy1, dummy2, hv_err;
+	u8 mss, *mssbuf;
+	int err;
+
+	err = -ENODEV;
+	if (!ldom_domaining_enabled)
+		goto out_err;
+
+	err = -EINVAL;
+	if (!cfgp)
+		goto out_err;
+	if (!name)
+		goto out_err;
+
+	switch (cfgp->mode) {
+	case LDC_MODE_RAW:
+		mops = &raw_ops;
+		mss = LDC_PACKET_SIZE;
+		break;
+
+	case LDC_MODE_UNRELIABLE:
+		mops = &nonraw_ops;
+		mss = LDC_PACKET_SIZE - 8;
+		break;
+
+	case LDC_MODE_STREAM:
+		mops = &stream_ops;
+		mss = LDC_PACKET_SIZE - 8 - 8;
+		break;
+
+	default:
+		goto out_err;
+	}
+
+	if (!cfgp->event || !event_arg || !cfgp->rx_irq || !cfgp->tx_irq)
+		goto out_err;
+
+	hv_err = sun4v_ldc_tx_qinfo(id, &dummy1, &dummy2);
+	err = -ENODEV;
+	if (hv_err == HV_ECHANNEL)
+		goto out_err;
+
+	err = -EEXIST;
+	if (__ldc_channel_exists(id))
+		goto out_err;
+
+	mssbuf = NULL;
+
+	lp = kzalloc(sizeof(*lp), GFP_KERNEL);
+	err = -ENOMEM;
+	if (!lp)
+		goto out_err;
+
+	spin_lock_init(&lp->lock);
+
+	lp->id = id;
+
+	err = ldc_iommu_init(name, lp);
+	if (err)
+		goto out_free_ldc;
+
+	lp->mops = mops;
+	lp->mss = mss;
+
+	lp->cfg = *cfgp;
+	if (!lp->cfg.mtu)
+		lp->cfg.mtu = LDC_DEFAULT_MTU;
+
+	if (lp->cfg.mode == LDC_MODE_STREAM) {
+		mssbuf = kzalloc(lp->cfg.mtu, GFP_KERNEL);
+		if (!mssbuf) {
+			err = -ENOMEM;
+			goto out_free_iommu;
+		}
+		lp->mssbuf = mssbuf;
+	}
+
+	lp->event_arg = event_arg;
+
+	/* XXX allow setting via ldc_channel_config to override defaults
+	 * XXX or use some formula based upon mtu
+	 */
+	lp->tx_num_entries = LDC_DEFAULT_NUM_ENTRIES;
+	lp->rx_num_entries = LDC_DEFAULT_NUM_ENTRIES;
+
+	err = alloc_queue("TX", lp->tx_num_entries,
+			  &lp->tx_base, &lp->tx_ra);
+	if (err)
+		goto out_free_mssbuf;
+
+	err = alloc_queue("RX", lp->rx_num_entries,
+			  &lp->rx_base, &lp->rx_ra);
+	if (err)
+		goto out_free_txq;
+
+	lp->flags |= LDC_FLAG_ALLOCED_QUEUES;
+
+	lp->hs_state = LDC_HS_CLOSED;
+	ldc_set_state(lp, LDC_STATE_INIT);
+
+	INIT_HLIST_NODE(&lp->list);
+	hlist_add_head(&lp->list, &ldc_channel_list);
+
+	INIT_HLIST_HEAD(&lp->mh_list);
+
+	snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name);
+	snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name);
+
+	err = request_irq(lp->cfg.rx_irq, ldc_rx, 0,
+			  lp->rx_irq_name, lp);
+	if (err)
+		goto out_free_txq;
+
+	err = request_irq(lp->cfg.tx_irq, ldc_tx, 0,
+			  lp->tx_irq_name, lp);
+	if (err) {
+		free_irq(lp->cfg.rx_irq, lp);
+		goto out_free_txq;
+	}
+
+	return lp;
+
+out_free_txq:
+	free_queue(lp->tx_num_entries, lp->tx_base);
+
+out_free_mssbuf:
+	kfree(mssbuf);
+
+out_free_iommu:
+	ldc_iommu_release(lp);
+
+out_free_ldc:
+	kfree(lp);
+
+out_err:
+	return ERR_PTR(err);
+}
+EXPORT_SYMBOL(ldc_alloc);
+
+void ldc_unbind(struct ldc_channel *lp)
+{
+	if (lp->flags & LDC_FLAG_REGISTERED_IRQS) {
+		free_irq(lp->cfg.rx_irq, lp);
+		free_irq(lp->cfg.tx_irq, lp);
+		lp->flags &= ~LDC_FLAG_REGISTERED_IRQS;
+	}
+
+	if (lp->flags & LDC_FLAG_REGISTERED_QUEUES) {
+		sun4v_ldc_tx_qconf(lp->id, 0, 0);
+		sun4v_ldc_rx_qconf(lp->id, 0, 0);
+		lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES;
+	}
+	if (lp->flags & LDC_FLAG_ALLOCED_QUEUES) {
+		free_queue(lp->tx_num_entries, lp->tx_base);
+		free_queue(lp->rx_num_entries, lp->rx_base);
+		lp->flags &= ~LDC_FLAG_ALLOCED_QUEUES;
+	}
+
+	ldc_set_state(lp, LDC_STATE_INIT);
+}
+EXPORT_SYMBOL(ldc_unbind);
+
+void ldc_free(struct ldc_channel *lp)
+{
+	ldc_unbind(lp);
+	hlist_del(&lp->list);
+	kfree(lp->mssbuf);
+	ldc_iommu_release(lp);
+
+	kfree(lp);
+}
+EXPORT_SYMBOL(ldc_free);
+
+/* Bind the channel.  This registers the LDC queues with
+ * the hypervisor and puts the channel into a pseudo-listening
+ * state.  This does not initiate a handshake, ldc_connect() does
+ * that.
+ */
+int ldc_bind(struct ldc_channel *lp)
+{
+	unsigned long hv_err, flags;
+	int err = -EINVAL;
+
+	if (lp->state != LDC_STATE_INIT)
+		return -EINVAL;
+
+	spin_lock_irqsave(&lp->lock, flags);
+
+	enable_irq(lp->cfg.rx_irq);
+	enable_irq(lp->cfg.tx_irq);
+
+	lp->flags |= LDC_FLAG_REGISTERED_IRQS;
+
+	err = -ENODEV;
+	hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0);
+	if (hv_err)
+		goto out_free_irqs;
+
+	hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
+	if (hv_err)
+		goto out_free_irqs;
+
+	hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0);
+	if (hv_err)
+		goto out_unmap_tx;
+
+	hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
+	if (hv_err)
+		goto out_unmap_tx;
+
+	lp->flags |= LDC_FLAG_REGISTERED_QUEUES;
+
+	hv_err = sun4v_ldc_tx_get_state(lp->id,
+					&lp->tx_head,
+					&lp->tx_tail,
+					&lp->chan_state);
+	err = -EBUSY;
+	if (hv_err)
+		goto out_unmap_rx;
+
+	lp->tx_acked = lp->tx_head;
+
+	lp->hs_state = LDC_HS_OPEN;
+	ldc_set_state(lp, LDC_STATE_BOUND);
+
+	if (lp->cfg.mode == LDC_MODE_RAW) {
+		/*
+		 * There is no handshake in RAW mode, so handshake
+		 * is completed.
+		 */
+		lp->hs_state = LDC_HS_COMPLETE;
+	}
+
+	spin_unlock_irqrestore(&lp->lock, flags);
+
+	return 0;
+
+out_unmap_rx:
+	lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES;
+	sun4v_ldc_rx_qconf(lp->id, 0, 0);
+
+out_unmap_tx:
+	sun4v_ldc_tx_qconf(lp->id, 0, 0);
+
+out_free_irqs:
+	lp->flags &= ~LDC_FLAG_REGISTERED_IRQS;
+	free_irq(lp->cfg.tx_irq, lp);
+	free_irq(lp->cfg.rx_irq, lp);
+
+	spin_unlock_irqrestore(&lp->lock, flags);
+
+	return err;
+}
+EXPORT_SYMBOL(ldc_bind);
+
+int ldc_connect(struct ldc_channel *lp)
+{
+	unsigned long flags;
+	int err;
+
+	if (lp->cfg.mode == LDC_MODE_RAW)
+		return -EINVAL;
+
+	spin_lock_irqsave(&lp->lock, flags);
+
+	if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) ||
+	    !(lp->flags & LDC_FLAG_REGISTERED_QUEUES) ||
+	    lp->hs_state != LDC_HS_OPEN)
+		err = ((lp->hs_state > LDC_HS_OPEN) ? 0 : -EINVAL);
+	else
+		err = start_handshake(lp);
+
+	spin_unlock_irqrestore(&lp->lock, flags);
+
+	return err;
+}
+EXPORT_SYMBOL(ldc_connect);
+
+int ldc_disconnect(struct ldc_channel *lp)
+{
+	unsigned long hv_err, flags;
+	int err;
+
+	if (lp->cfg.mode == LDC_MODE_RAW)
+		return -EINVAL;
+
+	if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) ||
+	    !(lp->flags & LDC_FLAG_REGISTERED_QUEUES))
+		return -EINVAL;
+
+	spin_lock_irqsave(&lp->lock, flags);
+
+	err = -ENODEV;
+	hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0);
+	if (hv_err)
+		goto out_err;
+
+	hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
+	if (hv_err)
+		goto out_err;
+
+	hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0);
+	if (hv_err)
+		goto out_err;
+
+	hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
+	if (hv_err)
+		goto out_err;
+
+	ldc_set_state(lp, LDC_STATE_BOUND);
+	lp->hs_state = LDC_HS_OPEN;
+	lp->flags |= LDC_FLAG_RESET;
+
+	spin_unlock_irqrestore(&lp->lock, flags);
+
+	return 0;
+
+out_err:
+	sun4v_ldc_tx_qconf(lp->id, 0, 0);
+	sun4v_ldc_rx_qconf(lp->id, 0, 0);
+	free_irq(lp->cfg.tx_irq, lp);
+	free_irq(lp->cfg.rx_irq, lp);
+	lp->flags &= ~(LDC_FLAG_REGISTERED_IRQS |
+		       LDC_FLAG_REGISTERED_QUEUES);
+	ldc_set_state(lp, LDC_STATE_INIT);
+
+	spin_unlock_irqrestore(&lp->lock, flags);
+
+	return err;
+}
+EXPORT_SYMBOL(ldc_disconnect);
+
+int ldc_state(struct ldc_channel *lp)
+{
+	return lp->state;
+}
+EXPORT_SYMBOL(ldc_state);
+
+void ldc_set_state(struct ldc_channel *lp, u8 state)
+{
+	ldcdbg(STATE, "STATE (%s) --> (%s)\n",
+	       state_to_str(lp->state),
+	       state_to_str(state));
+
+	lp->state = state;
+}
+EXPORT_SYMBOL(ldc_set_state);
+
+int ldc_mode(struct ldc_channel *lp)
+{
+	return lp->cfg.mode;
+}
+EXPORT_SYMBOL(ldc_mode);
+
+int ldc_rx_reset(struct ldc_channel *lp)
+{
+	return __set_rx_head(lp, lp->rx_tail);
+}
+EXPORT_SYMBOL(ldc_rx_reset);
+
+void __ldc_print(struct ldc_channel *lp, const char *caller)
+{
+	pr_info("%s: id=0x%lx flags=0x%x state=%s cstate=0x%lx hsstate=0x%x\n"
+		"\trx_h=0x%lx rx_t=0x%lx rx_n=%ld\n"
+		"\ttx_h=0x%lx tx_t=0x%lx tx_n=%ld\n"
+		"\trcv_nxt=%u snd_nxt=%u\n",
+		caller, lp->id, lp->flags, state_to_str(lp->state),
+		lp->chan_state, lp->hs_state,
+		lp->rx_head, lp->rx_tail, lp->rx_num_entries,
+		lp->tx_head, lp->tx_tail, lp->tx_num_entries,
+		lp->rcv_nxt, lp->snd_nxt);
+}
+EXPORT_SYMBOL(__ldc_print);
+
+static int write_raw(struct ldc_channel *lp, const void *buf, unsigned int size)
+{
+	struct ldc_packet *p;
+	unsigned long new_tail, hv_err;
+	int err;
+
+	hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail,
+					&lp->chan_state);
+	if (unlikely(hv_err))
+		return -EBUSY;
+
+	if (unlikely(lp->chan_state != LDC_CHANNEL_UP))
+		return LDC_ABORT(lp);
+
+	if (size > LDC_PACKET_SIZE)
+		return -EMSGSIZE;
+
+	p = data_get_tx_packet(lp, &new_tail);
+	if (!p)
+		return -EAGAIN;
+
+	memcpy(p, buf, size);
+
+	err = send_tx_packet(lp, p, new_tail);
+	if (!err)
+		err = size;
+
+	return err;
+}
+
+static int read_raw(struct ldc_channel *lp, void *buf, unsigned int size)
+{
+	struct ldc_packet *p;
+	unsigned long hv_err, new;
+	int err;
+
+	if (size < LDC_PACKET_SIZE)
+		return -EINVAL;
+
+	hv_err = sun4v_ldc_rx_get_state(lp->id,
+					&lp->rx_head,
+					&lp->rx_tail,
+					&lp->chan_state);
+	if (hv_err)
+		return LDC_ABORT(lp);
+
+	if (lp->chan_state == LDC_CHANNEL_DOWN ||
+	    lp->chan_state == LDC_CHANNEL_RESETTING)
+		return -ECONNRESET;
+
+	if (lp->rx_head == lp->rx_tail)
+		return 0;
+
+	p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE);
+	memcpy(buf, p, LDC_PACKET_SIZE);
+
+	new = rx_advance(lp, lp->rx_head);
+	lp->rx_head = new;
+
+	err = __set_rx_head(lp, new);
+	if (err < 0)
+		err = -ECONNRESET;
+	else
+		err = LDC_PACKET_SIZE;
+
+	return err;
+}
+
+static const struct ldc_mode_ops raw_ops = {
+	.write		=	write_raw,
+	.read		=	read_raw,
+};
+
+static int write_nonraw(struct ldc_channel *lp, const void *buf,
+			unsigned int size)
+{
+	unsigned long hv_err, tail;
+	unsigned int copied;
+	u32 seq;
+	int err;
+
+	hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail,
+					&lp->chan_state);
+	if (unlikely(hv_err))
+		return -EBUSY;
+
+	if (unlikely(lp->chan_state != LDC_CHANNEL_UP))
+		return LDC_ABORT(lp);
+
+	if (!tx_has_space_for(lp, size))
+		return -EAGAIN;
+
+	seq = lp->snd_nxt;
+	copied = 0;
+	tail = lp->tx_tail;
+	while (copied < size) {
+		struct ldc_packet *p = lp->tx_base + (tail / LDC_PACKET_SIZE);
+		u8 *data = ((lp->cfg.mode == LDC_MODE_UNRELIABLE) ?
+			    p->u.u_data :
+			    p->u.r.r_data);
+		int data_len;
+
+		p->type = LDC_DATA;
+		p->stype = LDC_INFO;
+		p->ctrl = 0;
+
+		data_len = size - copied;
+		if (data_len > lp->mss)
+			data_len = lp->mss;
+
+		BUG_ON(data_len > LDC_LEN);
+
+		p->env = (data_len |
+			  (copied == 0 ? LDC_START : 0) |
+			  (data_len == size - copied ? LDC_STOP : 0));
+
+		p->seqid = ++seq;
+
+		ldcdbg(DATA, "SENT DATA [%02x:%02x:%02x:%02x:%08x]\n",
+		       p->type,
+		       p->stype,
+		       p->ctrl,
+		       p->env,
+		       p->seqid);
+
+		memcpy(data, buf, data_len);
+		buf += data_len;
+		copied += data_len;
+
+		tail = tx_advance(lp, tail);
+	}
+
+	err = set_tx_tail(lp, tail);
+	if (!err) {
+		lp->snd_nxt = seq;
+		err = size;
+	}
+
+	return err;
+}
+
+static int rx_bad_seq(struct ldc_channel *lp, struct ldc_packet *p,
+		      struct ldc_packet *first_frag)
+{
+	int err;
+
+	if (first_frag)
+		lp->rcv_nxt = first_frag->seqid - 1;
+
+	err = send_data_nack(lp, p);
+	if (err)
+		return err;
+
+	err = ldc_rx_reset(lp);
+	if (err < 0)
+		return LDC_ABORT(lp);
+
+	return 0;
+}
+
+static int data_ack_nack(struct ldc_channel *lp, struct ldc_packet *p)
+{
+	if (p->stype & LDC_ACK) {
+		int err = process_data_ack(lp, p);
+		if (err)
+			return err;
+	}
+	if (p->stype & LDC_NACK)
+		return LDC_ABORT(lp);
+
+	return 0;
+}
+
+static int rx_data_wait(struct ldc_channel *lp, unsigned long cur_head)
+{
+	unsigned long dummy;
+	int limit = 1000;
+
+	ldcdbg(DATA, "DATA WAIT cur_head[%lx] rx_head[%lx] rx_tail[%lx]\n",
+	       cur_head, lp->rx_head, lp->rx_tail);
+	while (limit-- > 0) {
+		unsigned long hv_err;
+
+		hv_err = sun4v_ldc_rx_get_state(lp->id,
+						&dummy,
+						&lp->rx_tail,
+						&lp->chan_state);
+		if (hv_err)
+			return LDC_ABORT(lp);
+
+		if (lp->chan_state == LDC_CHANNEL_DOWN ||
+		    lp->chan_state == LDC_CHANNEL_RESETTING)
+			return -ECONNRESET;
+
+		if (cur_head != lp->rx_tail) {
+			ldcdbg(DATA, "DATA WAIT DONE "
+			       "head[%lx] tail[%lx] chan_state[%lx]\n",
+			       dummy, lp->rx_tail, lp->chan_state);
+			return 0;
+		}
+
+		udelay(1);
+	}
+	return -EAGAIN;
+}
+
+static int rx_set_head(struct ldc_channel *lp, unsigned long head)
+{
+	int err = __set_rx_head(lp, head);
+
+	if (err < 0)
+		return LDC_ABORT(lp);
+
+	lp->rx_head = head;
+	return 0;
+}
+
+static void send_data_ack(struct ldc_channel *lp)
+{
+	unsigned long new_tail;
+	struct ldc_packet *p;
+
+	p = data_get_tx_packet(lp, &new_tail);
+	if (likely(p)) {
+		int err;
+
+		memset(p, 0, sizeof(*p));
+		p->type = LDC_DATA;
+		p->stype = LDC_ACK;
+		p->ctrl = 0;
+		p->seqid = lp->snd_nxt + 1;
+		p->u.r.ackid = lp->rcv_nxt;
+
+		err = send_tx_packet(lp, p, new_tail);
+		if (!err)
+			lp->snd_nxt++;
+	}
+}
+
+static int read_nonraw(struct ldc_channel *lp, void *buf, unsigned int size)
+{
+	struct ldc_packet *first_frag;
+	unsigned long hv_err, new;
+	int err, copied;
+
+	hv_err = sun4v_ldc_rx_get_state(lp->id,
+					&lp->rx_head,
+					&lp->rx_tail,
+					&lp->chan_state);
+	if (hv_err)
+		return LDC_ABORT(lp);
+
+	if (lp->chan_state == LDC_CHANNEL_DOWN ||
+	    lp->chan_state == LDC_CHANNEL_RESETTING)
+		return -ECONNRESET;
+
+	if (lp->rx_head == lp->rx_tail)
+		return 0;
+
+	first_frag = NULL;
+	copied = err = 0;
+	new = lp->rx_head;
+	while (1) {
+		struct ldc_packet *p;
+		int pkt_len;
+
+		BUG_ON(new == lp->rx_tail);
+		p = lp->rx_base + (new / LDC_PACKET_SIZE);
+
+		ldcdbg(RX, "RX read pkt[%02x:%02x:%02x:%02x:%08x:%08x] "
+		       "rcv_nxt[%08x]\n",
+		       p->type,
+		       p->stype,
+		       p->ctrl,
+		       p->env,
+		       p->seqid,
+		       p->u.r.ackid,
+		       lp->rcv_nxt);
+
+		if (unlikely(!rx_seq_ok(lp, p->seqid))) {
+			err = rx_bad_seq(lp, p, first_frag);
+			copied = 0;
+			break;
+		}
+
+		if (p->type & LDC_CTRL) {
+			err = process_control_frame(lp, p);
+			if (err < 0)
+				break;
+			err = 0;
+		}
+
+		lp->rcv_nxt = p->seqid;
+
+		/*
+		 * If this is a control-only packet, there is nothing
+		 * else to do but advance the rx queue since the packet
+		 * was already processed above.
+		 */
+		if (!(p->type & LDC_DATA)) {
+			new = rx_advance(lp, new);
+			break;
+		}
+		if (p->stype & (LDC_ACK | LDC_NACK)) {
+			err = data_ack_nack(lp, p);
+			if (err)
+				break;
+		}
+		if (!(p->stype & LDC_INFO)) {
+			new = rx_advance(lp, new);
+			err = rx_set_head(lp, new);
+			if (err)
+				break;
+			goto no_data;
+		}
+
+		pkt_len = p->env & LDC_LEN;
+
+		/* Every initial packet starts with the START bit set.
+		 *
+		 * Singleton packets will have both START+STOP set.
+		 *
+		 * Fragments will have START set in the first frame, STOP
+		 * set in the last frame, and neither bit set in middle
+		 * frames of the packet.
+		 *
+		 * Therefore if we are at the beginning of a packet and
+		 * we don't see START, or we are in the middle of a fragmented
+		 * packet and do see START, we are unsynchronized and should
+		 * flush the RX queue.
+		 */
+		if ((first_frag == NULL && !(p->env & LDC_START)) ||
+		    (first_frag != NULL &&  (p->env & LDC_START))) {
+			if (!first_frag)
+				new = rx_advance(lp, new);
+
+			err = rx_set_head(lp, new);
+			if (err)
+				break;
+
+			if (!first_frag)
+				goto no_data;
+		}
+		if (!first_frag)
+			first_frag = p;
+
+		if (pkt_len > size - copied) {
+			/* User didn't give us a big enough buffer,
+			 * what to do?  This is a pretty serious error.
+			 *
+			 * Since we haven't updated the RX ring head to
+			 * consume any of the packets, signal the error
+			 * to the user and just leave the RX ring alone.
+			 *
+			 * This seems the best behavior because this allows
+			 * a user of the LDC layer to start with a small
+			 * RX buffer for ldc_read() calls and use -EMSGSIZE
+			 * as a cue to enlarge it's read buffer.
+			 */
+			err = -EMSGSIZE;
+			break;
+		}
+
+		/* Ok, we are gonna eat this one.  */
+		new = rx_advance(lp, new);
+
+		memcpy(buf,
+		       (lp->cfg.mode == LDC_MODE_UNRELIABLE ?
+			p->u.u_data : p->u.r.r_data), pkt_len);
+		buf += pkt_len;
+		copied += pkt_len;
+
+		if (p->env & LDC_STOP)
+			break;
+
+no_data:
+		if (new == lp->rx_tail) {
+			err = rx_data_wait(lp, new);
+			if (err)
+				break;
+		}
+	}
+
+	if (!err)
+		err = rx_set_head(lp, new);
+
+	if (err && first_frag)
+		lp->rcv_nxt = first_frag->seqid - 1;
+
+	if (!err) {
+		err = copied;
+		if (err > 0 && lp->cfg.mode != LDC_MODE_UNRELIABLE)
+			send_data_ack(lp);
+	}
+
+	return err;
+}
+
+static const struct ldc_mode_ops nonraw_ops = {
+	.write		=	write_nonraw,
+	.read		=	read_nonraw,
+};
+
+static int write_stream(struct ldc_channel *lp, const void *buf,
+			unsigned int size)
+{
+	if (size > lp->cfg.mtu)
+		size = lp->cfg.mtu;
+	return write_nonraw(lp, buf, size);
+}
+
+static int read_stream(struct ldc_channel *lp, void *buf, unsigned int size)
+{
+	if (!lp->mssbuf_len) {
+		int err = read_nonraw(lp, lp->mssbuf, lp->cfg.mtu);
+		if (err < 0)
+			return err;
+
+		lp->mssbuf_len = err;
+		lp->mssbuf_off = 0;
+	}
+
+	if (size > lp->mssbuf_len)
+		size = lp->mssbuf_len;
+	memcpy(buf, lp->mssbuf + lp->mssbuf_off, size);
+
+	lp->mssbuf_off += size;
+	lp->mssbuf_len -= size;
+
+	return size;
+}
+
+static const struct ldc_mode_ops stream_ops = {
+	.write		=	write_stream,
+	.read		=	read_stream,
+};
+
+int ldc_write(struct ldc_channel *lp, const void *buf, unsigned int size)
+{
+	unsigned long flags;
+	int err;
+
+	if (!buf)
+		return -EINVAL;
+
+	if (!size)
+		return 0;
+
+	spin_lock_irqsave(&lp->lock, flags);
+
+	if (lp->hs_state != LDC_HS_COMPLETE)
+		err = -ENOTCONN;
+	else
+		err = lp->mops->write(lp, buf, size);
+
+	spin_unlock_irqrestore(&lp->lock, flags);
+
+	return err;
+}
+EXPORT_SYMBOL(ldc_write);
+
+int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size)
+{
+	unsigned long flags;
+	int err;
+
+	ldcdbg(RX, "%s: entered size=%d\n", __func__, size);
+
+	if (!buf)
+		return -EINVAL;
+
+	if (!size)
+		return 0;
+
+	spin_lock_irqsave(&lp->lock, flags);
+
+	if (lp->hs_state != LDC_HS_COMPLETE)
+		err = -ENOTCONN;
+	else
+		err = lp->mops->read(lp, buf, size);
+
+	spin_unlock_irqrestore(&lp->lock, flags);
+
+	ldcdbg(RX, "%s: mode=%d, head=%lu, tail=%lu rv=%d\n", __func__,
+	       lp->cfg.mode, lp->rx_head, lp->rx_tail, err);
+
+	return err;
+}
+EXPORT_SYMBOL(ldc_read);
+
+static u64 pagesize_code(void)
+{
+	switch (PAGE_SIZE) {
+	default:
+	case (8ULL * 1024ULL):
+		return 0;
+	case (64ULL * 1024ULL):
+		return 1;
+	case (512ULL * 1024ULL):
+		return 2;
+	case (4ULL * 1024ULL * 1024ULL):
+		return 3;
+	case (32ULL * 1024ULL * 1024ULL):
+		return 4;
+	case (256ULL * 1024ULL * 1024ULL):
+		return 5;
+	}
+}
+
+static u64 make_cookie(u64 index, u64 pgsz_code, u64 page_offset)
+{
+	return ((pgsz_code << COOKIE_PGSZ_CODE_SHIFT) |
+		(index << PAGE_SHIFT) |
+		page_offset);
+}
+
+
+static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu,
+					     unsigned long npages)
+{
+	long entry;
+
+	entry = iommu_tbl_range_alloc(NULL, &iommu->iommu_map_table,
+				      npages, NULL, (unsigned long)-1, 0);
+	if (unlikely(entry == IOMMU_ERROR_CODE))
+		return NULL;
+
+	return iommu->page_table + entry;
+}
+
+static u64 perm_to_mte(unsigned int map_perm)
+{
+	u64 mte_base;
+
+	mte_base = pagesize_code();
+
+	if (map_perm & LDC_MAP_SHADOW) {
+		if (map_perm & LDC_MAP_R)
+			mte_base |= LDC_MTE_COPY_R;
+		if (map_perm & LDC_MAP_W)
+			mte_base |= LDC_MTE_COPY_W;
+	}
+	if (map_perm & LDC_MAP_DIRECT) {
+		if (map_perm & LDC_MAP_R)
+			mte_base |= LDC_MTE_READ;
+		if (map_perm & LDC_MAP_W)
+			mte_base |= LDC_MTE_WRITE;
+		if (map_perm & LDC_MAP_X)
+			mte_base |= LDC_MTE_EXEC;
+	}
+	if (map_perm & LDC_MAP_IO) {
+		if (map_perm & LDC_MAP_R)
+			mte_base |= LDC_MTE_IOMMU_R;
+		if (map_perm & LDC_MAP_W)
+			mte_base |= LDC_MTE_IOMMU_W;
+	}
+
+	return mte_base;
+}
+
+static int pages_in_region(unsigned long base, long len)
+{
+	int count = 0;
+
+	do {
+		unsigned long new = (base + PAGE_SIZE) & PAGE_MASK;
+
+		len -= (new - base);
+		base = new;
+		count++;
+	} while (len > 0);
+
+	return count;
+}
+
+struct cookie_state {
+	struct ldc_mtable_entry		*page_table;
+	struct ldc_trans_cookie		*cookies;
+	u64				mte_base;
+	u64				prev_cookie;
+	u32				pte_idx;
+	u32				nc;
+};
+
+static void fill_cookies(struct cookie_state *sp, unsigned long pa,
+			 unsigned long off, unsigned long len)
+{
+	do {
+		unsigned long tlen, new = pa + PAGE_SIZE;
+		u64 this_cookie;
+
+		sp->page_table[sp->pte_idx].mte = sp->mte_base | pa;
+
+		tlen = PAGE_SIZE;
+		if (off)
+			tlen = PAGE_SIZE - off;
+		if (tlen > len)
+			tlen = len;
+
+		this_cookie = make_cookie(sp->pte_idx,
+					  pagesize_code(), off);
+
+		off = 0;
+
+		if (this_cookie == sp->prev_cookie) {
+			sp->cookies[sp->nc - 1].cookie_size += tlen;
+		} else {
+			sp->cookies[sp->nc].cookie_addr = this_cookie;
+			sp->cookies[sp->nc].cookie_size = tlen;
+			sp->nc++;
+		}
+		sp->prev_cookie = this_cookie + tlen;
+
+		sp->pte_idx++;
+
+		len -= tlen;
+		pa = new;
+	} while (len > 0);
+}
+
+static int sg_count_one(struct scatterlist *sg)
+{
+	unsigned long base = page_to_pfn(sg_page(sg)) << PAGE_SHIFT;
+	long len = sg->length;
+
+	if ((sg->offset | len) & (8UL - 1))
+		return -EFAULT;
+
+	return pages_in_region(base + sg->offset, len);
+}
+
+static int sg_count_pages(struct scatterlist *sg, int num_sg)
+{
+	int count;
+	int i;
+
+	count = 0;
+	for (i = 0; i < num_sg; i++) {
+		int err = sg_count_one(sg + i);
+		if (err < 0)
+			return err;
+		count += err;
+	}
+
+	return count;
+}
+
+int ldc_map_sg(struct ldc_channel *lp,
+	       struct scatterlist *sg, int num_sg,
+	       struct ldc_trans_cookie *cookies, int ncookies,
+	       unsigned int map_perm)
+{
+	unsigned long i, npages;
+	struct ldc_mtable_entry *base;
+	struct cookie_state state;
+	struct ldc_iommu *iommu;
+	int err;
+	struct scatterlist *s;
+
+	if (map_perm & ~LDC_MAP_ALL)
+		return -EINVAL;
+
+	err = sg_count_pages(sg, num_sg);
+	if (err < 0)
+		return err;
+
+	npages = err;
+	if (err > ncookies)
+		return -EMSGSIZE;
+
+	iommu = &lp->iommu;
+
+	base = alloc_npages(iommu, npages);
+
+	if (!base)
+		return -ENOMEM;
+
+	state.page_table = iommu->page_table;
+	state.cookies = cookies;
+	state.mte_base = perm_to_mte(map_perm);
+	state.prev_cookie = ~(u64)0;
+	state.pte_idx = (base - iommu->page_table);
+	state.nc = 0;
+
+	for_each_sg(sg, s, num_sg, i) {
+		fill_cookies(&state, page_to_pfn(sg_page(s)) << PAGE_SHIFT,
+			     s->offset, s->length);
+	}
+
+	return state.nc;
+}
+EXPORT_SYMBOL(ldc_map_sg);
+
+int ldc_map_single(struct ldc_channel *lp,
+		   void *buf, unsigned int len,
+		   struct ldc_trans_cookie *cookies, int ncookies,
+		   unsigned int map_perm)
+{
+	unsigned long npages, pa;
+	struct ldc_mtable_entry *base;
+	struct cookie_state state;
+	struct ldc_iommu *iommu;
+
+	if ((map_perm & ~LDC_MAP_ALL) || (ncookies < 1))
+		return -EINVAL;
+
+	pa = __pa(buf);
+	if ((pa | len) & (8UL - 1))
+		return -EFAULT;
+
+	npages = pages_in_region(pa, len);
+
+	iommu = &lp->iommu;
+
+	base = alloc_npages(iommu, npages);
+
+	if (!base)
+		return -ENOMEM;
+
+	state.page_table = iommu->page_table;
+	state.cookies = cookies;
+	state.mte_base = perm_to_mte(map_perm);
+	state.prev_cookie = ~(u64)0;
+	state.pte_idx = (base - iommu->page_table);
+	state.nc = 0;
+	fill_cookies(&state, (pa & PAGE_MASK), (pa & ~PAGE_MASK), len);
+	BUG_ON(state.nc > ncookies);
+
+	return state.nc;
+}
+EXPORT_SYMBOL(ldc_map_single);
+
+
+static void free_npages(unsigned long id, struct ldc_iommu *iommu,
+			u64 cookie, u64 size)
+{
+	unsigned long npages, entry;
+
+	npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT;
+
+	entry = ldc_cookie_to_index(cookie, iommu);
+	ldc_demap(iommu, id, cookie, entry, npages);
+	iommu_tbl_range_free(&iommu->iommu_map_table, cookie, npages, entry);
+}
+
+void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies,
+	       int ncookies)
+{
+	struct ldc_iommu *iommu = &lp->iommu;
+	int i;
+	unsigned long flags;
+
+	spin_lock_irqsave(&iommu->lock, flags);
+	for (i = 0; i < ncookies; i++) {
+		u64 addr = cookies[i].cookie_addr;
+		u64 size = cookies[i].cookie_size;
+
+		free_npages(lp->id, iommu, addr, size);
+	}
+	spin_unlock_irqrestore(&iommu->lock, flags);
+}
+EXPORT_SYMBOL(ldc_unmap);
+
+int ldc_copy(struct ldc_channel *lp, int copy_dir,
+	     void *buf, unsigned int len, unsigned long offset,
+	     struct ldc_trans_cookie *cookies, int ncookies)
+{
+	unsigned int orig_len;
+	unsigned long ra;
+	int i;
+
+	if (copy_dir != LDC_COPY_IN && copy_dir != LDC_COPY_OUT) {
+		printk(KERN_ERR PFX "ldc_copy: ID[%lu] Bad copy_dir[%d]\n",
+		       lp->id, copy_dir);
+		return -EINVAL;
+	}
+
+	ra = __pa(buf);
+	if ((ra | len | offset) & (8UL - 1)) {
+		printk(KERN_ERR PFX "ldc_copy: ID[%lu] Unaligned buffer "
+		       "ra[%lx] len[%x] offset[%lx]\n",
+		       lp->id, ra, len, offset);
+		return -EFAULT;
+	}
+
+	if (lp->hs_state != LDC_HS_COMPLETE ||
+	    (lp->flags & LDC_FLAG_RESET)) {
+		printk(KERN_ERR PFX "ldc_copy: ID[%lu] Link down hs_state[%x] "
+		       "flags[%x]\n", lp->id, lp->hs_state, lp->flags);
+		return -ECONNRESET;
+	}
+
+	orig_len = len;
+	for (i = 0; i < ncookies; i++) {
+		unsigned long cookie_raddr = cookies[i].cookie_addr;
+		unsigned long this_len = cookies[i].cookie_size;
+		unsigned long actual_len;
+
+		if (unlikely(offset)) {
+			unsigned long this_off = offset;
+
+			if (this_off > this_len)
+				this_off = this_len;
+
+			offset -= this_off;
+			this_len -= this_off;
+			if (!this_len)
+				continue;
+			cookie_raddr += this_off;
+		}
+
+		if (this_len > len)
+			this_len = len;
+
+		while (1) {
+			unsigned long hv_err;
+
+			hv_err = sun4v_ldc_copy(lp->id, copy_dir,
+						cookie_raddr, ra,
+						this_len, &actual_len);
+			if (unlikely(hv_err)) {
+				printk(KERN_ERR PFX "ldc_copy: ID[%lu] "
+				       "HV error %lu\n",
+				       lp->id, hv_err);
+				if (lp->hs_state != LDC_HS_COMPLETE ||
+				    (lp->flags & LDC_FLAG_RESET))
+					return -ECONNRESET;
+				else
+					return -EFAULT;
+			}
+
+			cookie_raddr += actual_len;
+			ra += actual_len;
+			len -= actual_len;
+			if (actual_len == this_len)
+				break;
+
+			this_len -= actual_len;
+		}
+
+		if (!len)
+			break;
+	}
+
+	/* It is caller policy what to do about short copies.
+	 * For example, a networking driver can declare the
+	 * packet a runt and drop it.
+	 */
+
+	return orig_len - len;
+}
+EXPORT_SYMBOL(ldc_copy);
+
+void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len,
+			  struct ldc_trans_cookie *cookies, int *ncookies,
+			  unsigned int map_perm)
+{
+	void *buf;
+	int err;
+
+	if (len & (8UL - 1))
+		return ERR_PTR(-EINVAL);
+
+	buf = kzalloc(len, GFP_ATOMIC);
+	if (!buf)
+		return ERR_PTR(-ENOMEM);
+
+	err = ldc_map_single(lp, buf, len, cookies, *ncookies, map_perm);
+	if (err < 0) {
+		kfree(buf);
+		return ERR_PTR(err);
+	}
+	*ncookies = err;
+
+	return buf;
+}
+EXPORT_SYMBOL(ldc_alloc_exp_dring);
+
+void ldc_free_exp_dring(struct ldc_channel *lp, void *buf, unsigned int len,
+			struct ldc_trans_cookie *cookies, int ncookies)
+{
+	ldc_unmap(lp, cookies, ncookies);
+	kfree(buf);
+}
+EXPORT_SYMBOL(ldc_free_exp_dring);
+
+static int __init ldc_init(void)
+{
+	unsigned long major, minor;
+	struct mdesc_handle *hp;
+	const u64 *v;
+	int err;
+	u64 mp;
+
+	hp = mdesc_grab();
+	if (!hp)
+		return -ENODEV;
+
+	mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform");
+	err = -ENODEV;
+	if (mp == MDESC_NODE_NULL)
+		goto out;
+
+	v = mdesc_get_property(hp, mp, "domaining-enabled", NULL);
+	if (!v)
+		goto out;
+
+	major = 1;
+	minor = 0;
+	if (sun4v_hvapi_register(HV_GRP_LDOM, major, &minor)) {
+		printk(KERN_INFO PFX "Could not register LDOM hvapi.\n");
+		goto out;
+	}
+
+	printk(KERN_INFO "%s", version);
+
+	if (!*v) {
+		printk(KERN_INFO PFX "Domaining disabled.\n");
+		goto out;
+	}
+	ldom_domaining_enabled = 1;
+	err = 0;
+
+out:
+	mdesc_release(hp);
+	return err;
+}
+
+core_initcall(ldc_init);