Adding upstream version 6.1.76.upstream/6.1.76

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

9 files changed, 4943 insertions, 0 deletions

diff --git a/drivers/char/xillybus/Kconfig b/drivers/char/xillybus/Kconfig
new file mode 100644
index 000000000..a8036dad4
--- /dev/null
+++ b/drivers/char/xillybus/Kconfig
@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Xillybus devices
+#
+
+config XILLYBUS_CLASS
+	tristate
+
+config XILLYBUS
+	tristate "Xillybus generic FPGA interface"
+	depends on PCI || OF
+	select CRC32
+	select XILLYBUS_CLASS
+	help
+	  Xillybus is a generic interface for peripherals designed on
+	  programmable logic (FPGA). The driver probes the hardware for
+	  its capabilities, and creates device files accordingly.
+
+	  If unsure, say N.
+
+if XILLYBUS
+
+config XILLYBUS_PCIE
+	tristate "Xillybus over PCIe"
+	depends on PCI_MSI
+	help
+	  Set to M if you want Xillybus to use PCI Express for communicating
+	  with the FPGA. The module will be called xillybus_pcie.
+
+config XILLYBUS_OF
+	tristate "Xillybus over Device Tree"
+	depends on OF && HAS_DMA
+	help
+	  Set to M if you want Xillybus to find its resources from the
+	  Open Firmware Flattened Device Tree. If the target is an embedded
+	  system, say M. The module will be called xillybus_of.
+
+endif # if XILLYBUS
+
+# XILLYUSB doesn't depend on XILLYBUS
+
+config XILLYUSB
+	tristate "XillyUSB: Xillybus generic FPGA interface for USB"
+	depends on USB
+	select CRC32
+	select XILLYBUS_CLASS
+	help
+	  XillyUSB is the Xillybus variant which uses USB for communicating
+	  with the FPGA.
+
+	  Set to M if you want Xillybus to use USB for communicating with
+	  the FPGA. The module will be called xillyusb.
diff --git a/drivers/char/xillybus/Makefile b/drivers/char/xillybus/Makefile
new file mode 100644
index 000000000..16f31d032
--- /dev/null
+++ b/drivers/char/xillybus/Makefile
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Makefile for Xillybus driver
+#
+
+obj-$(CONFIG_XILLYBUS_CLASS)	+= xillybus_class.o
+obj-$(CONFIG_XILLYBUS)		+= xillybus_core.o
+obj-$(CONFIG_XILLYBUS_PCIE)	+= xillybus_pcie.o
+obj-$(CONFIG_XILLYBUS_OF)	+= xillybus_of.o
+obj-$(CONFIG_XILLYUSB)		+= xillyusb.o
diff --git a/drivers/char/xillybus/xillybus.h b/drivers/char/xillybus/xillybus.h
new file mode 100644
index 000000000..51de7cbc5
--- /dev/null
+++ b/drivers/char/xillybus/xillybus.h
@@ -0,0 +1,126 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * linux/drivers/misc/xillybus.h
+ *
+ * Copyright 2011 Xillybus Ltd, http://xillybus.com
+ *
+ * Header file for the Xillybus FPGA/host framework.
+ */
+
+#ifndef __XILLYBUS_H
+#define __XILLYBUS_H
+
+#include <linux/list.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/sched.h>
+#include <linux/cdev.h>
+#include <linux/spinlock.h>
+#include <linux/mutex.h>
+#include <linux/workqueue.h>
+
+struct xilly_endpoint_hardware;
+
+struct xilly_buffer {
+	void *addr;
+	dma_addr_t dma_addr;
+	int end_offset; /* Counting elements, not bytes */
+};
+
+struct xilly_idt_handle {
+	unsigned char *chandesc;
+	unsigned char *names;
+	int names_len;
+	int entries;
+};
+
+/*
+ * Read-write confusion: wr_* and rd_* notation sticks to FPGA view, so
+ * wr_* buffers are those consumed by read(), since the FPGA writes to them
+ * and vice versa.
+ */
+
+struct xilly_channel {
+	struct xilly_endpoint *endpoint;
+	int chan_num;
+	int log2_element_size;
+	int seekable;
+
+	struct xilly_buffer **wr_buffers; /* FPGA writes, driver reads! */
+	int num_wr_buffers;
+	unsigned int wr_buf_size; /* In bytes */
+	int wr_fpga_buf_idx;
+	int wr_host_buf_idx;
+	int wr_host_buf_pos;
+	int wr_empty;
+	int wr_ready; /* Significant only when wr_empty == 1 */
+	int wr_sleepy;
+	int wr_eof;
+	int wr_hangup;
+	spinlock_t wr_spinlock;
+	struct mutex wr_mutex;
+	wait_queue_head_t wr_wait;
+	wait_queue_head_t wr_ready_wait;
+	int wr_ref_count;
+	int wr_synchronous;
+	int wr_allow_partial;
+	int wr_exclusive_open;
+	int wr_supports_nonempty;
+
+	struct xilly_buffer **rd_buffers; /* FPGA reads, driver writes! */
+	int num_rd_buffers;
+	unsigned int rd_buf_size; /* In bytes */
+	int rd_fpga_buf_idx;
+	int rd_host_buf_pos;
+	int rd_host_buf_idx;
+	int rd_full;
+	spinlock_t rd_spinlock;
+	struct mutex rd_mutex;
+	wait_queue_head_t rd_wait;
+	int rd_ref_count;
+	int rd_allow_partial;
+	int rd_synchronous;
+	int rd_exclusive_open;
+	struct delayed_work rd_workitem;
+	unsigned char rd_leftovers[4];
+};
+
+struct xilly_endpoint {
+	struct device *dev;
+	struct module *owner;
+
+	int dma_using_dac; /* =1 if 64-bit DMA is used, =0 otherwise. */
+	__iomem void *registers;
+	int fatal_error;
+
+	struct mutex register_mutex;
+	wait_queue_head_t ep_wait;
+
+	int num_channels; /* EXCLUDING message buffer */
+	struct xilly_channel **channels;
+	int msg_counter;
+	int failed_messages;
+	int idtlen;
+
+	u32 *msgbuf_addr;
+	dma_addr_t msgbuf_dma_addr;
+	unsigned int msg_buf_size;
+};
+
+struct xilly_mapping {
+	struct device *device;
+	dma_addr_t dma_addr;
+	size_t size;
+	int direction;
+};
+
+irqreturn_t xillybus_isr(int irq, void *data);
+
+struct xilly_endpoint *xillybus_init_endpoint(struct device *dev);
+
+int xillybus_endpoint_discovery(struct xilly_endpoint *endpoint);
+
+void xillybus_endpoint_remove(struct xilly_endpoint *endpoint);
+
+#endif /* __XILLYBUS_H */
diff --git a/drivers/char/xillybus/xillybus_class.c b/drivers/char/xillybus/xillybus_class.c
new file mode 100644
index 000000000..0f238648d
--- /dev/null
+++ b/drivers/char/xillybus/xillybus_class.c
@@ -0,0 +1,260 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright 2021 Xillybus Ltd, http://xillybus.com
+ *
+ * Driver for the Xillybus class
+ */
+
+#include <linux/types.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/fs.h>
+#include <linux/cdev.h>
+#include <linux/slab.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+
+#include "xillybus_class.h"
+
+MODULE_DESCRIPTION("Driver for Xillybus class");
+MODULE_AUTHOR("Eli Billauer, Xillybus Ltd.");
+MODULE_ALIAS("xillybus_class");
+MODULE_LICENSE("GPL v2");
+
+static DEFINE_MUTEX(unit_mutex);
+static LIST_HEAD(unit_list);
+static struct class *xillybus_class;
+
+#define UNITNAMELEN 16
+
+struct xilly_unit {
+	struct list_head list_entry;
+	void *private_data;
+
+	struct cdev *cdev;
+	char name[UNITNAMELEN];
+	int major;
+	int lowest_minor;
+	int num_nodes;
+};
+
+int xillybus_init_chrdev(struct device *dev,
+			 const struct file_operations *fops,
+			 struct module *owner,
+			 void *private_data,
+			 unsigned char *idt, unsigned int len,
+			 int num_nodes,
+			 const char *prefix, bool enumerate)
+{
+	int rc;
+	dev_t mdev;
+	int i;
+	char devname[48];
+
+	struct device *device;
+	size_t namelen;
+	struct xilly_unit *unit, *u;
+
+	unit = kzalloc(sizeof(*unit), GFP_KERNEL);
+
+	if (!unit)
+		return -ENOMEM;
+
+	mutex_lock(&unit_mutex);
+
+	if (!enumerate)
+		snprintf(unit->name, UNITNAMELEN, "%s", prefix);
+
+	for (i = 0; enumerate; i++) {
+		snprintf(unit->name, UNITNAMELEN, "%s_%02d",
+			 prefix, i);
+
+		enumerate = false;
+		list_for_each_entry(u, &unit_list, list_entry)
+			if (!strcmp(unit->name, u->name)) {
+				enumerate = true;
+				break;
+			}
+	}
+
+	rc = alloc_chrdev_region(&mdev, 0, num_nodes, unit->name);
+
+	if (rc) {
+		dev_warn(dev, "Failed to obtain major/minors");
+		goto fail_obtain;
+	}
+
+	unit->major = MAJOR(mdev);
+	unit->lowest_minor = MINOR(mdev);
+	unit->num_nodes = num_nodes;
+	unit->private_data = private_data;
+
+	unit->cdev = cdev_alloc();
+	if (!unit->cdev) {
+		rc = -ENOMEM;
+		goto unregister_chrdev;
+	}
+	unit->cdev->ops = fops;
+	unit->cdev->owner = owner;
+
+	rc = cdev_add(unit->cdev, MKDEV(unit->major, unit->lowest_minor),
+		      unit->num_nodes);
+	if (rc) {
+		dev_err(dev, "Failed to add cdev.\n");
+		/* kobject_put() is normally done by cdev_del() */
+		kobject_put(&unit->cdev->kobj);
+		goto unregister_chrdev;
+	}
+
+	for (i = 0; i < num_nodes; i++) {
+		namelen = strnlen(idt, len);
+
+		if (namelen == len) {
+			dev_err(dev, "IDT's list of names is too short. This is exceptionally weird, because its CRC is OK\n");
+			rc = -ENODEV;
+			goto unroll_device_create;
+		}
+
+		snprintf(devname, sizeof(devname), "%s_%s",
+			 unit->name, idt);
+
+		len -= namelen + 1;
+		idt += namelen + 1;
+
+		device = device_create(xillybus_class,
+				       NULL,
+				       MKDEV(unit->major,
+					     i + unit->lowest_minor),
+				       NULL,
+				       "%s", devname);
+
+		if (IS_ERR(device)) {
+			dev_err(dev, "Failed to create %s device. Aborting.\n",
+				devname);
+			rc = -ENODEV;
+			goto unroll_device_create;
+		}
+	}
+
+	if (len) {
+		dev_err(dev, "IDT's list of names is too long. This is exceptionally weird, because its CRC is OK\n");
+		rc = -ENODEV;
+		goto unroll_device_create;
+	}
+
+	list_add_tail(&unit->list_entry, &unit_list);
+
+	dev_info(dev, "Created %d device files.\n", num_nodes);
+
+	mutex_unlock(&unit_mutex);
+
+	return 0;
+
+unroll_device_create:
+	for (i--; i >= 0; i--)
+		device_destroy(xillybus_class, MKDEV(unit->major,
+						     i + unit->lowest_minor));
+
+	cdev_del(unit->cdev);
+
+unregister_chrdev:
+	unregister_chrdev_region(MKDEV(unit->major, unit->lowest_minor),
+				 unit->num_nodes);
+
+fail_obtain:
+	mutex_unlock(&unit_mutex);
+
+	kfree(unit);
+
+	return rc;
+}
+EXPORT_SYMBOL(xillybus_init_chrdev);
+
+void xillybus_cleanup_chrdev(void *private_data,
+			     struct device *dev)
+{
+	int minor;
+	struct xilly_unit *unit = NULL, *iter;
+
+	mutex_lock(&unit_mutex);
+
+	list_for_each_entry(iter, &unit_list, list_entry)
+		if (iter->private_data == private_data) {
+			unit = iter;
+			break;
+		}
+
+	if (!unit) {
+		dev_err(dev, "Weird bug: Failed to find unit\n");
+		mutex_unlock(&unit_mutex);
+		return;
+	}
+
+	for (minor = unit->lowest_minor;
+	     minor < (unit->lowest_minor + unit->num_nodes);
+	     minor++)
+		device_destroy(xillybus_class, MKDEV(unit->major, minor));
+
+	cdev_del(unit->cdev);
+
+	unregister_chrdev_region(MKDEV(unit->major, unit->lowest_minor),
+				 unit->num_nodes);
+
+	dev_info(dev, "Removed %d device files.\n",
+		 unit->num_nodes);
+
+	list_del(&unit->list_entry);
+	kfree(unit);
+
+	mutex_unlock(&unit_mutex);
+}
+EXPORT_SYMBOL(xillybus_cleanup_chrdev);
+
+int xillybus_find_inode(struct inode *inode,
+			void **private_data, int *index)
+{
+	int minor = iminor(inode);
+	int major = imajor(inode);
+	struct xilly_unit *unit = NULL, *iter;
+
+	mutex_lock(&unit_mutex);
+
+	list_for_each_entry(iter, &unit_list, list_entry)
+		if (iter->major == major &&
+		    minor >= iter->lowest_minor &&
+		    minor < (iter->lowest_minor + iter->num_nodes)) {
+			unit = iter;
+			break;
+		}
+
+	mutex_unlock(&unit_mutex);
+
+	if (!unit)
+		return -ENODEV;
+
+	*private_data = unit->private_data;
+	*index = minor - unit->lowest_minor;
+
+	return 0;
+}
+EXPORT_SYMBOL(xillybus_find_inode);
+
+static int __init xillybus_class_init(void)
+{
+	xillybus_class = class_create(THIS_MODULE, "xillybus");
+
+	if (IS_ERR(xillybus_class)) {
+		pr_warn("Failed to register xillybus class\n");
+
+		return PTR_ERR(xillybus_class);
+	}
+	return 0;
+}
+
+static void __exit xillybus_class_exit(void)
+{
+	class_destroy(xillybus_class);
+}
+
+module_init(xillybus_class_init);
+module_exit(xillybus_class_exit);
diff --git a/drivers/char/xillybus/xillybus_class.h b/drivers/char/xillybus/xillybus_class.h
new file mode 100644
index 000000000..5dbfdfc95
--- /dev/null
+++ b/drivers/char/xillybus/xillybus_class.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright 2021 Xillybus Ltd, http://www.xillybus.com
+ *
+ * Header file for the Xillybus class
+ */
+
+#ifndef __XILLYBUS_CLASS_H
+#define __XILLYBUS_CLASS_H
+
+#include <linux/types.h>
+#include <linux/device.h>
+#include <linux/fs.h>
+#include <linux/module.h>
+
+int xillybus_init_chrdev(struct device *dev,
+			 const struct file_operations *fops,
+			 struct module *owner,
+			 void *private_data,
+			 unsigned char *idt, unsigned int len,
+			 int num_nodes,
+			 const char *prefix, bool enumerate);
+
+void xillybus_cleanup_chrdev(void *private_data,
+			     struct device *dev);
+
+int xillybus_find_inode(struct inode *inode,
+			void **private_data, int *index);
+
+#endif /* __XILLYBUS_CLASS_H */
diff --git a/drivers/char/xillybus/xillybus_core.c b/drivers/char/xillybus/xillybus_core.c
new file mode 100644
index 000000000..11b7c4749
--- /dev/null
+++ b/drivers/char/xillybus/xillybus_core.c
@@ -0,0 +1,1991 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * linux/drivers/misc/xillybus_core.c
+ *
+ * Copyright 2011 Xillybus Ltd, http://xillybus.com
+ *
+ * Driver for the Xillybus FPGA/host framework.
+ *
+ * This driver interfaces with a special IP core in an FPGA, setting up
+ * a pipe between a hardware FIFO in the programmable logic and a device
+ * file in the host. The number of such pipes and their attributes are
+ * set up on the logic. This driver detects these automatically and
+ * creates the device files accordingly.
+ */
+
+#include <linux/list.h>
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/io.h>
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/sched.h>
+#include <linux/fs.h>
+#include <linux/spinlock.h>
+#include <linux/mutex.h>
+#include <linux/crc32.h>
+#include <linux/poll.h>
+#include <linux/delay.h>
+#include <linux/slab.h>
+#include <linux/workqueue.h>
+#include "xillybus.h"
+#include "xillybus_class.h"
+
+MODULE_DESCRIPTION("Xillybus core functions");
+MODULE_AUTHOR("Eli Billauer, Xillybus Ltd.");
+MODULE_ALIAS("xillybus_core");
+MODULE_LICENSE("GPL v2");
+
+/* General timeout is 100 ms, rx timeout is 10 ms */
+#define XILLY_RX_TIMEOUT (10*HZ/1000)
+#define XILLY_TIMEOUT (100*HZ/1000)
+
+#define fpga_msg_ctrl_reg              0x0008
+#define fpga_dma_control_reg           0x0020
+#define fpga_dma_bufno_reg             0x0024
+#define fpga_dma_bufaddr_lowaddr_reg   0x0028
+#define fpga_dma_bufaddr_highaddr_reg  0x002c
+#define fpga_buf_ctrl_reg              0x0030
+#define fpga_buf_offset_reg            0x0034
+#define fpga_endian_reg                0x0040
+
+#define XILLYMSG_OPCODE_RELEASEBUF 1
+#define XILLYMSG_OPCODE_QUIESCEACK 2
+#define XILLYMSG_OPCODE_FIFOEOF 3
+#define XILLYMSG_OPCODE_FATAL_ERROR 4
+#define XILLYMSG_OPCODE_NONEMPTY 5
+
+static const char xillyname[] = "xillybus";
+
+static struct workqueue_struct *xillybus_wq;
+
+/*
+ * Locking scheme: Mutexes protect invocations of character device methods.
+ * If both locks are taken, wr_mutex is taken first, rd_mutex second.
+ *
+ * wr_spinlock protects wr_*_buf_idx, wr_empty, wr_sleepy, wr_ready and the
+ * buffers' end_offset fields against changes made by IRQ handler (and in
+ * theory, other file request handlers, but the mutex handles that). Nothing
+ * else.
+ * They are held for short direct memory manipulations. Needless to say,
+ * no mutex locking is allowed when a spinlock is held.
+ *
+ * rd_spinlock does the same with rd_*_buf_idx, rd_empty and end_offset.
+ *
+ * register_mutex is endpoint-specific, and is held when non-atomic
+ * register operations are performed. wr_mutex and rd_mutex may be
+ * held when register_mutex is taken, but none of the spinlocks. Note that
+ * register_mutex doesn't protect against sporadic buf_ctrl_reg writes
+ * which are unrelated to buf_offset_reg, since they are harmless.
+ *
+ * Blocking on the wait queues is allowed with mutexes held, but not with
+ * spinlocks.
+ *
+ * Only interruptible blocking is allowed on mutexes and wait queues.
+ *
+ * All in all, the locking order goes (with skips allowed, of course):
+ * wr_mutex -> rd_mutex -> register_mutex -> wr_spinlock -> rd_spinlock
+ */
+
+static void malformed_message(struct xilly_endpoint *endpoint, u32 *buf)
+{
+	int opcode;
+	int msg_channel, msg_bufno, msg_data, msg_dir;
+
+	opcode = (buf[0] >> 24) & 0xff;
+	msg_dir = buf[0] & 1;
+	msg_channel = (buf[0] >> 1) & 0x7ff;
+	msg_bufno = (buf[0] >> 12) & 0x3ff;
+	msg_data = buf[1] & 0xfffffff;
+
+	dev_warn(endpoint->dev,
+		 "Malformed message (skipping): opcode=%d, channel=%03x, dir=%d, bufno=%03x, data=%07x\n",
+		 opcode, msg_channel, msg_dir, msg_bufno, msg_data);
+}
+
+/*
+ * xillybus_isr assumes the interrupt is allocated exclusively to it,
+ * which is the natural case MSI and several other hardware-oriented
+ * interrupts. Sharing is not allowed.
+ */
+
+irqreturn_t xillybus_isr(int irq, void *data)
+{
+	struct xilly_endpoint *ep = data;
+	u32 *buf;
+	unsigned int buf_size;
+	int i;
+	int opcode;
+	unsigned int msg_channel, msg_bufno, msg_data, msg_dir;
+	struct xilly_channel *channel;
+
+	buf = ep->msgbuf_addr;
+	buf_size = ep->msg_buf_size/sizeof(u32);
+
+	dma_sync_single_for_cpu(ep->dev, ep->msgbuf_dma_addr,
+				ep->msg_buf_size, DMA_FROM_DEVICE);
+
+	for (i = 0; i < buf_size; i += 2) {
+		if (((buf[i+1] >> 28) & 0xf) != ep->msg_counter) {
+			malformed_message(ep, &buf[i]);
+			dev_warn(ep->dev,
+				 "Sending a NACK on counter %x (instead of %x) on entry %d\n",
+				 ((buf[i+1] >> 28) & 0xf),
+				 ep->msg_counter,
+				 i/2);
+
+			if (++ep->failed_messages > 10) {
+				dev_err(ep->dev,
+					"Lost sync with interrupt messages. Stopping.\n");
+			} else {
+				dma_sync_single_for_device(ep->dev,
+							   ep->msgbuf_dma_addr,
+							   ep->msg_buf_size,
+							   DMA_FROM_DEVICE);
+
+				iowrite32(0x01,  /* Message NACK */
+					  ep->registers + fpga_msg_ctrl_reg);
+			}
+			return IRQ_HANDLED;
+		} else if (buf[i] & (1 << 22)) /* Last message */
+			break;
+	}
+
+	if (i >= buf_size) {
+		dev_err(ep->dev, "Bad interrupt message. Stopping.\n");
+		return IRQ_HANDLED;
+	}
+
+	buf_size = i + 2;
+
+	for (i = 0; i < buf_size; i += 2) { /* Scan through messages */
+		opcode = (buf[i] >> 24) & 0xff;
+
+		msg_dir = buf[i] & 1;
+		msg_channel = (buf[i] >> 1) & 0x7ff;
+		msg_bufno = (buf[i] >> 12) & 0x3ff;
+		msg_data = buf[i+1] & 0xfffffff;
+
+		switch (opcode) {
+		case XILLYMSG_OPCODE_RELEASEBUF:
+			if ((msg_channel > ep->num_channels) ||
+			    (msg_channel == 0)) {
+				malformed_message(ep, &buf[i]);
+				break;
+			}
+
+			channel = ep->channels[msg_channel];
+
+			if (msg_dir) { /* Write channel */
+				if (msg_bufno >= channel->num_wr_buffers) {
+					malformed_message(ep, &buf[i]);
+					break;
+				}
+				spin_lock(&channel->wr_spinlock);
+				channel->wr_buffers[msg_bufno]->end_offset =
+					msg_data;
+				channel->wr_fpga_buf_idx = msg_bufno;
+				channel->wr_empty = 0;
+				channel->wr_sleepy = 0;
+				spin_unlock(&channel->wr_spinlock);
+
+				wake_up_interruptible(&channel->wr_wait);
+
+			} else {
+				/* Read channel */
+
+				if (msg_bufno >= channel->num_rd_buffers) {
+					malformed_message(ep, &buf[i]);
+					break;
+				}
+
+				spin_lock(&channel->rd_spinlock);
+				channel->rd_fpga_buf_idx = msg_bufno;
+				channel->rd_full = 0;
+				spin_unlock(&channel->rd_spinlock);
+
+				wake_up_interruptible(&channel->rd_wait);
+				if (!channel->rd_synchronous)
+					queue_delayed_work(
+						xillybus_wq,
+						&channel->rd_workitem,
+						XILLY_RX_TIMEOUT);
+			}
+
+			break;
+		case XILLYMSG_OPCODE_NONEMPTY:
+			if ((msg_channel > ep->num_channels) ||
+			    (msg_channel == 0) || (!msg_dir) ||
+			    !ep->channels[msg_channel]->wr_supports_nonempty) {
+				malformed_message(ep, &buf[i]);
+				break;
+			}
+
+			channel = ep->channels[msg_channel];
+
+			if (msg_bufno >= channel->num_wr_buffers) {
+				malformed_message(ep, &buf[i]);
+				break;
+			}
+			spin_lock(&channel->wr_spinlock);
+			if (msg_bufno == channel->wr_host_buf_idx)
+				channel->wr_ready = 1;
+			spin_unlock(&channel->wr_spinlock);
+
+			wake_up_interruptible(&channel->wr_ready_wait);
+
+			break;
+		case XILLYMSG_OPCODE_QUIESCEACK:
+			ep->idtlen = msg_data;
+			wake_up_interruptible(&ep->ep_wait);
+
+			break;
+		case XILLYMSG_OPCODE_FIFOEOF:
+			if ((msg_channel > ep->num_channels) ||
+			    (msg_channel == 0) || (!msg_dir) ||
+			    !ep->channels[msg_channel]->num_wr_buffers) {
+				malformed_message(ep, &buf[i]);
+				break;
+			}
+			channel = ep->channels[msg_channel];
+			spin_lock(&channel->wr_spinlock);
+			channel->wr_eof = msg_bufno;
+			channel->wr_sleepy = 0;
+
+			channel->wr_hangup = channel->wr_empty &&
+				(channel->wr_host_buf_idx == msg_bufno);
+
+			spin_unlock(&channel->wr_spinlock);
+
+			wake_up_interruptible(&channel->wr_wait);
+
+			break;
+		case XILLYMSG_OPCODE_FATAL_ERROR:
+			ep->fatal_error = 1;
+			wake_up_interruptible(&ep->ep_wait); /* For select() */
+			dev_err(ep->dev,
+				"FPGA reported a fatal error. This means that the low-level communication with the device has failed. This hardware problem is most likely unrelated to Xillybus (neither kernel module nor FPGA core), but reports are still welcome. All I/O is aborted.\n");
+			break;
+		default:
+			malformed_message(ep, &buf[i]);
+			break;
+		}
+	}
+
+	dma_sync_single_for_device(ep->dev, ep->msgbuf_dma_addr,
+				   ep->msg_buf_size, DMA_FROM_DEVICE);
+
+	ep->msg_counter = (ep->msg_counter + 1) & 0xf;
+	ep->failed_messages = 0;
+	iowrite32(0x03, ep->registers + fpga_msg_ctrl_reg); /* Message ACK */
+
+	return IRQ_HANDLED;
+}
+EXPORT_SYMBOL(xillybus_isr);
+
+/*
+ * A few trivial memory management functions.
+ * NOTE: These functions are used only on probe and remove, and therefore
+ * no locks are applied!
+ */
+
+static void xillybus_autoflush(struct work_struct *work);
+
+struct xilly_alloc_state {
+	void *salami;
+	int left_of_salami;
+	int nbuffer;
+	enum dma_data_direction direction;
+	u32 regdirection;
+};
+
+static void xilly_unmap(void *ptr)
+{
+	struct xilly_mapping *data = ptr;
+
+	dma_unmap_single(data->device, data->dma_addr,
+			 data->size, data->direction);
+
+	kfree(ptr);
+}
+
+static int xilly_map_single(struct xilly_endpoint *ep,
+			    void *ptr,
+			    size_t size,
+			    int direction,
+			    dma_addr_t *ret_dma_handle
+	)
+{
+	dma_addr_t addr;
+	struct xilly_mapping *this;
+
+	this = kzalloc(sizeof(*this), GFP_KERNEL);
+	if (!this)
+		return -ENOMEM;
+
+	addr = dma_map_single(ep->dev, ptr, size, direction);
+
+	if (dma_mapping_error(ep->dev, addr)) {
+		kfree(this);
+		return -ENODEV;
+	}
+
+	this->device = ep->dev;
+	this->dma_addr = addr;
+	this->size = size;
+	this->direction = direction;
+
+	*ret_dma_handle = addr;
+
+	return devm_add_action_or_reset(ep->dev, xilly_unmap, this);
+}
+
+static int xilly_get_dma_buffers(struct xilly_endpoint *ep,
+				 struct xilly_alloc_state *s,
+				 struct xilly_buffer **buffers,
+				 int bufnum, int bytebufsize)
+{
+	int i, rc;
+	dma_addr_t dma_addr;
+	struct device *dev = ep->dev;
+	struct xilly_buffer *this_buffer = NULL; /* Init to silence warning */
+
+	if (buffers) { /* Not the message buffer */
+		this_buffer = devm_kcalloc(dev, bufnum,
+					   sizeof(struct xilly_buffer),
+					   GFP_KERNEL);
+		if (!this_buffer)
+			return -ENOMEM;
+	}
+
+	for (i = 0; i < bufnum; i++) {
+		/*
+		 * Buffers are expected in descending size order, so there
+		 * is either enough space for this buffer or none at all.
+		 */
+
+		if ((s->left_of_salami < bytebufsize) &&
+		    (s->left_of_salami > 0)) {
+			dev_err(ep->dev,
+				"Corrupt buffer allocation in IDT. Aborting.\n");
+			return -ENODEV;
+		}
+
+		if (s->left_of_salami == 0) {
+			int allocorder, allocsize;
+
+			allocsize = PAGE_SIZE;
+			allocorder = 0;
+			while (bytebufsize > allocsize) {
+				allocsize *= 2;
+				allocorder++;
+			}
+
+			s->salami = (void *) devm_get_free_pages(
+				dev,
+				GFP_KERNEL | __GFP_DMA32 | __GFP_ZERO,
+				allocorder);
+			if (!s->salami)
+				return -ENOMEM;
+
+			s->left_of_salami = allocsize;
+		}
+
+		rc = xilly_map_single(ep, s->salami,
+				      bytebufsize, s->direction,
+				      &dma_addr);
+		if (rc)
+			return rc;
+
+		iowrite32((u32) (dma_addr & 0xffffffff),
+			  ep->registers + fpga_dma_bufaddr_lowaddr_reg);
+		iowrite32(((u32) ((((u64) dma_addr) >> 32) & 0xffffffff)),
+			  ep->registers + fpga_dma_bufaddr_highaddr_reg);
+
+		if (buffers) { /* Not the message buffer */
+			this_buffer->addr = s->salami;
+			this_buffer->dma_addr = dma_addr;
+			buffers[i] = this_buffer++;
+
+			iowrite32(s->regdirection | s->nbuffer++,
+				  ep->registers + fpga_dma_bufno_reg);
+		} else {
+			ep->msgbuf_addr = s->salami;
+			ep->msgbuf_dma_addr = dma_addr;
+			ep->msg_buf_size = bytebufsize;
+
+			iowrite32(s->regdirection,
+				  ep->registers + fpga_dma_bufno_reg);
+		}
+
+		s->left_of_salami -= bytebufsize;
+		s->salami += bytebufsize;
+	}
+	return 0;
+}
+
+static int xilly_setupchannels(struct xilly_endpoint *ep,
+			       unsigned char *chandesc,
+			       int entries)
+{
+	struct device *dev = ep->dev;
+	int i, entry, rc;
+	struct xilly_channel *channel;
+	int channelnum, bufnum, bufsize, format, is_writebuf;
+	int bytebufsize;
+	int synchronous, allowpartial, exclusive_open, seekable;
+	int supports_nonempty;
+	int msg_buf_done = 0;
+
+	struct xilly_alloc_state rd_alloc = {
+		.salami = NULL,
+		.left_of_salami = 0,
+		.nbuffer = 1,
+		.direction = DMA_TO_DEVICE,
+		.regdirection = 0,
+	};
+
+	struct xilly_alloc_state wr_alloc = {
+		.salami = NULL,
+		.left_of_salami = 0,
+		.nbuffer = 1,
+		.direction = DMA_FROM_DEVICE,
+		.regdirection = 0x80000000,
+	};
+
+	channel = devm_kcalloc(dev, ep->num_channels,
+			       sizeof(struct xilly_channel), GFP_KERNEL);
+	if (!channel)
+		return -ENOMEM;
+
+	ep->channels = devm_kcalloc(dev, ep->num_channels + 1,
+				    sizeof(struct xilly_channel *),
+				    GFP_KERNEL);
+	if (!ep->channels)
+		return -ENOMEM;
+
+	ep->channels[0] = NULL; /* Channel 0 is message buf. */
+
+	/* Initialize all channels with defaults */
+
+	for (i = 1; i <= ep->num_channels; i++) {
+		channel->wr_buffers = NULL;
+		channel->rd_buffers = NULL;
+		channel->num_wr_buffers = 0;
+		channel->num_rd_buffers = 0;
+		channel->wr_fpga_buf_idx = -1;
+		channel->wr_host_buf_idx = 0;
+		channel->wr_host_buf_pos = 0;
+		channel->wr_empty = 1;
+		channel->wr_ready = 0;
+		channel->wr_sleepy = 1;
+		channel->rd_fpga_buf_idx = 0;
+		channel->rd_host_buf_idx = 0;
+		channel->rd_host_buf_pos = 0;
+		channel->rd_full = 0;
+		channel->wr_ref_count = 0;
+		channel->rd_ref_count = 0;
+
+		spin_lock_init(&channel->wr_spinlock);
+		spin_lock_init(&channel->rd_spinlock);
+		mutex_init(&channel->wr_mutex);
+		mutex_init(&channel->rd_mutex);
+		init_waitqueue_head(&channel->rd_wait);
+		init_waitqueue_head(&channel->wr_wait);
+		init_waitqueue_head(&channel->wr_ready_wait);
+
+		INIT_DELAYED_WORK(&channel->rd_workitem, xillybus_autoflush);
+
+		channel->endpoint = ep;
+		channel->chan_num = i;
+
+		channel->log2_element_size = 0;
+
+		ep->channels[i] = channel++;
+	}
+
+	for (entry = 0; entry < entries; entry++, chandesc += 4) {
+		struct xilly_buffer **buffers = NULL;
+
+		is_writebuf = chandesc[0] & 0x01;
+		channelnum = (chandesc[0] >> 1) | ((chandesc[1] & 0x0f) << 7);
+		format = (chandesc[1] >> 4) & 0x03;
+		allowpartial = (chandesc[1] >> 6) & 0x01;
+		synchronous = (chandesc[1] >> 7) & 0x01;
+		bufsize = 1 << (chandesc[2] & 0x1f);
+		bufnum = 1 << (chandesc[3] & 0x0f);
+		exclusive_open = (chandesc[2] >> 7) & 0x01;
+		seekable = (chandesc[2] >> 6) & 0x01;
+		supports_nonempty = (chandesc[2] >> 5) & 0x01;
+
+		if ((channelnum > ep->num_channels) ||
+		    ((channelnum == 0) && !is_writebuf)) {
+			dev_err(ep->dev,
+				"IDT requests channel out of range. Aborting.\n");
+			return -ENODEV;
+		}
+
+		channel = ep->channels[channelnum]; /* NULL for msg channel */
+
+		if (!is_writebuf || channelnum > 0) {
+			channel->log2_element_size = ((format > 2) ?
+						      2 : format);
+
+			bytebufsize = bufsize *
+				(1 << channel->log2_element_size);
+
+			buffers = devm_kcalloc(dev, bufnum,
+					       sizeof(struct xilly_buffer *),
+					       GFP_KERNEL);
+			if (!buffers)
+				return -ENOMEM;
+		} else {
+			bytebufsize = bufsize << 2;
+		}
+
+		if (!is_writebuf) {
+			channel->num_rd_buffers = bufnum;
+			channel->rd_buf_size = bytebufsize;
+			channel->rd_allow_partial = allowpartial;
+			channel->rd_synchronous = synchronous;
+			channel->rd_exclusive_open = exclusive_open;
+			channel->seekable = seekable;
+
+			channel->rd_buffers = buffers;
+			rc = xilly_get_dma_buffers(ep, &rd_alloc, buffers,
+						   bufnum, bytebufsize);
+		} else if (channelnum > 0) {
+			channel->num_wr_buffers = bufnum;
+			channel->wr_buf_size = bytebufsize;
+
+			channel->seekable = seekable;
+			channel->wr_supports_nonempty = supports_nonempty;
+
+			channel->wr_allow_partial = allowpartial;
+			channel->wr_synchronous = synchronous;
+			channel->wr_exclusive_open = exclusive_open;
+
+			channel->wr_buffers = buffers;
+			rc = xilly_get_dma_buffers(ep, &wr_alloc, buffers,
+						   bufnum, bytebufsize);
+		} else {
+			rc = xilly_get_dma_buffers(ep, &wr_alloc, NULL,
+						   bufnum, bytebufsize);
+			msg_buf_done++;
+		}
+
+		if (rc)
+			return -ENOMEM;
+	}
+
+	if (!msg_buf_done) {
+		dev_err(ep->dev,
+			"Corrupt IDT: No message buffer. Aborting.\n");
+		return -ENODEV;
+	}
+	return 0;
+}
+
+static int xilly_scan_idt(struct xilly_endpoint *endpoint,
+			  struct xilly_idt_handle *idt_handle)
+{
+	int count = 0;
+	unsigned char *idt = endpoint->channels[1]->wr_buffers[0]->addr;
+	unsigned char *end_of_idt = idt + endpoint->idtlen - 4;
+	unsigned char *scan;
+	int len;
+
+	scan = idt + 1;
+	idt_handle->names = scan;
+
+	while ((scan <= end_of_idt) && *scan) {
+		while ((scan <= end_of_idt) && *scan++)
+			/* Do nothing, just scan thru string */;
+		count++;
+	}
+
+	idt_handle->names_len = scan - idt_handle->names;
+
+	scan++;
+
+	if (scan > end_of_idt) {
+		dev_err(endpoint->dev,
+			"IDT device name list overflow. Aborting.\n");
+		return -ENODEV;
+	}
+	idt_handle->chandesc = scan;
+
+	len = endpoint->idtlen - (3 + ((int) (scan - idt)));
+
+	if (len & 0x03) {
+		dev_err(endpoint->dev,
+			"Corrupt IDT device name list. Aborting.\n");
+		return -ENODEV;
+	}
+
+	idt_handle->entries = len >> 2;
+	endpoint->num_channels = count;
+
+	return 0;
+}
+
+static int xilly_obtain_idt(struct xilly_endpoint *endpoint)
+{
+	struct xilly_channel *channel;
+	unsigned char *version;
+	long t;
+
+	channel = endpoint->channels[1]; /* This should be generated ad-hoc */
+
+	channel->wr_sleepy = 1;
+
+	iowrite32(1 |
+		  (3 << 24), /* Opcode 3 for channel 0 = Send IDT */
+		  endpoint->registers + fpga_buf_ctrl_reg);
+
+	t = wait_event_interruptible_timeout(channel->wr_wait,
+					     (!channel->wr_sleepy),
+					     XILLY_TIMEOUT);
+
+	if (t <= 0) {
+		dev_err(endpoint->dev, "Failed to obtain IDT. Aborting.\n");
+
+		if (endpoint->fatal_error)
+			return -EIO;
+
+		return -ENODEV;
+	}
+
+	dma_sync_single_for_cpu(channel->endpoint->dev,
+				channel->wr_buffers[0]->dma_addr,
+				channel->wr_buf_size,
+				DMA_FROM_DEVICE);
+
+	if (channel->wr_buffers[0]->end_offset != endpoint->idtlen) {
+		dev_err(endpoint->dev,
+			"IDT length mismatch (%d != %d). Aborting.\n",
+			channel->wr_buffers[0]->end_offset, endpoint->idtlen);
+		return -ENODEV;
+	}
+
+	if (crc32_le(~0, channel->wr_buffers[0]->addr,
+		     endpoint->idtlen+1) != 0) {
+		dev_err(endpoint->dev, "IDT failed CRC check. Aborting.\n");
+		return -ENODEV;
+	}
+
+	version = channel->wr_buffers[0]->addr;
+
+	/* Check version number. Reject anything above 0x82. */
+	if (*version > 0x82) {
+		dev_err(endpoint->dev,
+			"No support for IDT version 0x%02x. Maybe the xillybus driver needs an upgrade. Aborting.\n",
+			*version);
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static ssize_t xillybus_read(struct file *filp, char __user *userbuf,
+			     size_t count, loff_t *f_pos)
+{
+	ssize_t rc;
+	unsigned long flags;
+	int bytes_done = 0;
+	int no_time_left = 0;
+	long deadline, left_to_sleep;
+	struct xilly_channel *channel = filp->private_data;
+
+	int empty, reached_eof, exhausted, ready;
+	/* Initializations are there only to silence warnings */
+
+	int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;
+	int waiting_bufidx;
+
+	if (channel->endpoint->fatal_error)
+		return -EIO;
+
+	deadline = jiffies + 1 + XILLY_RX_TIMEOUT;
+
+	rc = mutex_lock_interruptible(&channel->wr_mutex);
+	if (rc)
+		return rc;
+
+	while (1) { /* Note that we may drop mutex within this loop */
+		int bytes_to_do = count - bytes_done;
+
+		spin_lock_irqsave(&channel->wr_spinlock, flags);
+
+		empty = channel->wr_empty;
+		ready = !empty || channel->wr_ready;
+
+		if (!empty) {
+			bufidx = channel->wr_host_buf_idx;
+			bufpos = channel->wr_host_buf_pos;
+			howmany = ((channel->wr_buffers[bufidx]->end_offset
+				    + 1) << channel->log2_element_size)
+				- bufpos;
+
+			/* Update wr_host_* to its post-operation state */
+			if (howmany > bytes_to_do) {
+				bufferdone = 0;
+
+				howmany = bytes_to_do;
+				channel->wr_host_buf_pos += howmany;
+			} else {
+				bufferdone = 1;
+
+				channel->wr_host_buf_pos = 0;
+
+				if (bufidx == channel->wr_fpga_buf_idx) {
+					channel->wr_empty = 1;
+					channel->wr_sleepy = 1;
+					channel->wr_ready = 0;
+				}
+
+				if (bufidx >= (channel->num_wr_buffers - 1))
+					channel->wr_host_buf_idx = 0;
+				else
+					channel->wr_host_buf_idx++;
+			}
+		}
+
+		/*
+		 * Marking our situation after the possible changes above,
+		 * for use after releasing the spinlock.
+		 *
+		 * empty = empty before change
+		 * exhasted = empty after possible change
+		 */
+
+		reached_eof = channel->wr_empty &&
+			(channel->wr_host_buf_idx == channel->wr_eof);
+		channel->wr_hangup = reached_eof;
+		exhausted = channel->wr_empty;
+		waiting_bufidx = channel->wr_host_buf_idx;
+
+		spin_unlock_irqrestore(&channel->wr_spinlock, flags);
+
+		if (!empty) { /* Go on, now without the spinlock */
+
+			if (bufpos == 0) /* Position zero means it's virgin */
+				dma_sync_single_for_cpu(channel->endpoint->dev,
+							channel->wr_buffers[bufidx]->dma_addr,
+							channel->wr_buf_size,
+							DMA_FROM_DEVICE);
+
+			if (copy_to_user(
+				    userbuf,
+				    channel->wr_buffers[bufidx]->addr
+				    + bufpos, howmany))
+				rc = -EFAULT;
+
+			userbuf += howmany;
+			bytes_done += howmany;
+
+			if (bufferdone) {
+				dma_sync_single_for_device(channel->endpoint->dev,
+							   channel->wr_buffers[bufidx]->dma_addr,
+							   channel->wr_buf_size,
+							   DMA_FROM_DEVICE);
+
+				/*
+				 * Tell FPGA the buffer is done with. It's an
+				 * atomic operation to the FPGA, so what
+				 * happens with other channels doesn't matter,
+				 * and the certain channel is protected with
+				 * the channel-specific mutex.
+				 */
+
+				iowrite32(1 | (channel->chan_num << 1) |
+					  (bufidx << 12),
+					  channel->endpoint->registers +
+					  fpga_buf_ctrl_reg);
+			}
+
+			if (rc) {
+				mutex_unlock(&channel->wr_mutex);
+				return rc;
+			}
+		}
+
+		/* This includes a zero-count return = EOF */
+		if ((bytes_done >= count) || reached_eof)
+			break;
+
+		if (!exhausted)
+			continue; /* More in RAM buffer(s)? Just go on. */
+
+		if ((bytes_done > 0) &&
+		    (no_time_left ||
+		     (channel->wr_synchronous && channel->wr_allow_partial)))
+			break;
+
+		/*
+		 * Nonblocking read: The "ready" flag tells us that the FPGA
+		 * has data to send. In non-blocking mode, if it isn't on,
+		 * just return. But if there is, we jump directly to the point
+		 * where we ask for the FPGA to send all it has, and wait
+		 * until that data arrives. So in a sense, we *do* block in
+		 * nonblocking mode, but only for a very short time.
+		 */
+
+		if (!no_time_left && (filp->f_flags & O_NONBLOCK)) {
+			if (bytes_done > 0)
+				break;
+
+			if (ready)
+				goto desperate;
+
+			rc = -EAGAIN;
+			break;
+		}
+
+		if (!no_time_left || (bytes_done > 0)) {
+			/*
+			 * Note that in case of an element-misaligned read
+			 * request, offsetlimit will include the last element,
+			 * which will be partially read from.
+			 */
+			int offsetlimit = ((count - bytes_done) - 1) >>
+				channel->log2_element_size;
+			int buf_elements = channel->wr_buf_size >>
+				channel->log2_element_size;
+
+			/*
+			 * In synchronous mode, always send an offset limit.
+			 * Just don't send a value too big.
+			 */
+
+			if (channel->wr_synchronous) {
+				/* Don't request more than one buffer */
+				if (channel->wr_allow_partial &&
+				    (offsetlimit >= buf_elements))
+					offsetlimit = buf_elements - 1;
+
+				/* Don't request more than all buffers */
+				if (!channel->wr_allow_partial &&
+				    (offsetlimit >=
+				     (buf_elements * channel->num_wr_buffers)))
+					offsetlimit = buf_elements *
+						channel->num_wr_buffers - 1;
+			}
+
+			/*
+			 * In asynchronous mode, force early flush of a buffer
+			 * only if that will allow returning a full count. The
+			 * "offsetlimit < ( ... )" rather than "<=" excludes
+			 * requesting a full buffer, which would obviously
+			 * cause a buffer transmission anyhow
+			 */
+
+			if (channel->wr_synchronous ||
+			    (offsetlimit < (buf_elements - 1))) {
+				mutex_lock(&channel->endpoint->register_mutex);
+
+				iowrite32(offsetlimit,
+					  channel->endpoint->registers +
+					  fpga_buf_offset_reg);
+
+				iowrite32(1 | (channel->chan_num << 1) |
+					  (2 << 24) |  /* 2 = offset limit */
+					  (waiting_bufidx << 12),
+					  channel->endpoint->registers +
+					  fpga_buf_ctrl_reg);
+
+				mutex_unlock(&channel->endpoint->
+					     register_mutex);
+			}
+		}
+
+		/*
+		 * If partial completion is disallowed, there is no point in
+		 * timeout sleeping. Neither if no_time_left is set and
+		 * there's no data.
+		 */
+
+		if (!channel->wr_allow_partial ||
+		    (no_time_left && (bytes_done == 0))) {
+			/*
+			 * This do-loop will run more than once if another
+			 * thread reasserted wr_sleepy before we got the mutex
+			 * back, so we try again.
+			 */
+
+			do {
+				mutex_unlock(&channel->wr_mutex);
+
+				if (wait_event_interruptible(
+					    channel->wr_wait,
+					    (!channel->wr_sleepy)))
+					goto interrupted;
+
+				if (mutex_lock_interruptible(
+					    &channel->wr_mutex))
+					goto interrupted;
+			} while (channel->wr_sleepy);
+
+			continue;
+
+interrupted: /* Mutex is not held if got here */
+			if (channel->endpoint->fatal_error)
+				return -EIO;
+			if (bytes_done)
+				return bytes_done;
+			if (filp->f_flags & O_NONBLOCK)
+				return -EAGAIN; /* Don't admit snoozing */
+			return -EINTR;
+		}
+
+		left_to_sleep = deadline - ((long) jiffies);
+
+		/*
+		 * If our time is out, skip the waiting. We may miss wr_sleepy
+		 * being deasserted but hey, almost missing the train is like
+		 * missing it.
+		 */
+
+		if (left_to_sleep > 0) {
+			left_to_sleep =
+				wait_event_interruptible_timeout(
+					channel->wr_wait,
+					(!channel->wr_sleepy),
+					left_to_sleep);
+
+			if (left_to_sleep > 0) /* wr_sleepy deasserted */
+				continue;
+
+			if (left_to_sleep < 0) { /* Interrupt */
+				mutex_unlock(&channel->wr_mutex);
+				if (channel->endpoint->fatal_error)
+					return -EIO;
+				if (bytes_done)
+					return bytes_done;
+				return -EINTR;
+			}
+		}
+
+desperate:
+		no_time_left = 1; /* We're out of sleeping time. Desperate! */
+
+		if (bytes_done == 0) {
+			/*
+			 * Reaching here means that we allow partial return,
+			 * that we've run out of time, and that we have
+			 * nothing to return.
+			 * So tell the FPGA to send anything it has or gets.
+			 */
+
+			iowrite32(1 | (channel->chan_num << 1) |
+				  (3 << 24) |  /* Opcode 3, flush it all! */
+				  (waiting_bufidx << 12),
+				  channel->endpoint->registers +
+				  fpga_buf_ctrl_reg);
+		}
+
+		/*
+		 * Reaching here means that we *do* have data in the buffer,
+		 * but the "partial" flag disallows returning less than
+		 * required. And we don't have as much. So loop again,
+		 * which is likely to end up blocking indefinitely until
+		 * enough data has arrived.
+		 */
+	}
+
+	mutex_unlock(&channel->wr_mutex);
+
+	if (channel->endpoint->fatal_error)
+		return -EIO;
+
+	if (rc)
+		return rc;
+
+	return bytes_done;
+}
+
+/*
+ * The timeout argument takes values as follows:
+ *  >0 : Flush with timeout
+ * ==0 : Flush, and wait idefinitely for the flush to complete
+ *  <0 : Autoflush: Flush only if there's a single buffer occupied
+ */
+
+static int xillybus_myflush(struct xilly_channel *channel, long timeout)
+{
+	int rc;
+	unsigned long flags;
+
+	int end_offset_plus1;
+	int bufidx, bufidx_minus1;
+	int i;
+	int empty;
+	int new_rd_host_buf_pos;
+
+	if (channel->endpoint->fatal_error)
+		return -EIO;
+	rc = mutex_lock_interruptible(&channel->rd_mutex);
+	if (rc)
+		return rc;
+
+	/*
+	 * Don't flush a closed channel. This can happen when the work queued
+	 * autoflush thread fires off after the file has closed. This is not
+	 * an error, just something to dismiss.
+	 */
+
+	if (!channel->rd_ref_count)
+		goto done;
+
+	bufidx = channel->rd_host_buf_idx;
+
+	bufidx_minus1 = (bufidx == 0) ?
+		channel->num_rd_buffers - 1 :
+		bufidx - 1;
+
+	end_offset_plus1 = channel->rd_host_buf_pos >>
+		channel->log2_element_size;
+
+	new_rd_host_buf_pos = channel->rd_host_buf_pos -
+		(end_offset_plus1 << channel->log2_element_size);
+
+	/* Submit the current buffer if it's nonempty */
+	if (end_offset_plus1) {
+		unsigned char *tail = channel->rd_buffers[bufidx]->addr +
+			(end_offset_plus1 << channel->log2_element_size);
+
+		/* Copy  unflushed data, so we can put it in next buffer */
+		for (i = 0; i < new_rd_host_buf_pos; i++)
+			channel->rd_leftovers[i] = *tail++;
+
+		spin_lock_irqsave(&channel->rd_spinlock, flags);
+
+		/* Autoflush only if a single buffer is occupied */
+
+		if ((timeout < 0) &&
+		    (channel->rd_full ||
+		     (bufidx_minus1 != channel->rd_fpga_buf_idx))) {
+			spin_unlock_irqrestore(&channel->rd_spinlock, flags);
+			/*
+			 * A new work item may be queued by the ISR exactly
+			 * now, since the execution of a work item allows the
+			 * queuing of a new one while it's running.
+			 */
+			goto done;
+		}
+
+		/* The 4th element is never needed for data, so it's a flag */
+		channel->rd_leftovers[3] = (new_rd_host_buf_pos != 0);
+
+		/* Set up rd_full to reflect a certain moment's state */
+
+		if (bufidx == channel->rd_fpga_buf_idx)
+			channel->rd_full = 1;
+		spin_unlock_irqrestore(&channel->rd_spinlock, flags);
+
+		if (bufidx >= (channel->num_rd_buffers - 1))
+			channel->rd_host_buf_idx = 0;
+		else
+			channel->rd_host_buf_idx++;
+
+		dma_sync_single_for_device(channel->endpoint->dev,
+					   channel->rd_buffers[bufidx]->dma_addr,
+					   channel->rd_buf_size,
+					   DMA_TO_DEVICE);
+
+		mutex_lock(&channel->endpoint->register_mutex);
+
+		iowrite32(end_offset_plus1 - 1,
+			  channel->endpoint->registers + fpga_buf_offset_reg);
+
+		iowrite32((channel->chan_num << 1) | /* Channel ID */
+			  (2 << 24) |  /* Opcode 2, submit buffer */
+			  (bufidx << 12),
+			  channel->endpoint->registers + fpga_buf_ctrl_reg);
+
+		mutex_unlock(&channel->endpoint->register_mutex);
+	} else if (bufidx == 0) {
+		bufidx = channel->num_rd_buffers - 1;
+	} else {
+		bufidx--;
+	}
+
+	channel->rd_host_buf_pos = new_rd_host_buf_pos;
+
+	if (timeout < 0)
+		goto done; /* Autoflush */
+
+	/*
+	 * bufidx is now the last buffer written to (or equal to
+	 * rd_fpga_buf_idx if buffer was never written to), and
+	 * channel->rd_host_buf_idx the one after it.
+	 *
+	 * If bufidx == channel->rd_fpga_buf_idx we're either empty or full.
+	 */
+
+	while (1) { /* Loop waiting for draining of buffers */
+		spin_lock_irqsave(&channel->rd_spinlock, flags);
+
+		if (bufidx != channel->rd_fpga_buf_idx)
+			channel->rd_full = 1; /*
+					       * Not really full,
+					       * but needs waiting.
+					       */
+
+		empty = !channel->rd_full;
+
+		spin_unlock_irqrestore(&channel->rd_spinlock, flags);
+
+		if (empty)
+			break;
+
+		/*
+		 * Indefinite sleep with mutex taken. With data waiting for
+		 * flushing user should not be surprised if open() for write
+		 * sleeps.
+		 */
+		if (timeout == 0)
+			wait_event_interruptible(channel->rd_wait,
+						 (!channel->rd_full));
+
+		else if (wait_event_interruptible_timeout(
+				 channel->rd_wait,
+				 (!channel->rd_full),
+				 timeout) == 0) {
+			dev_warn(channel->endpoint->dev,
+				 "Timed out while flushing. Output data may be lost.\n");
+
+			rc = -ETIMEDOUT;
+			break;
+		}
+
+		if (channel->rd_full) {
+			rc = -EINTR;
+			break;
+		}
+	}
+
+done:
+	mutex_unlock(&channel->rd_mutex);
+
+	if (channel->endpoint->fatal_error)
+		return -EIO;
+
+	return rc;
+}
+
+static int xillybus_flush(struct file *filp, fl_owner_t id)
+{
+	if (!(filp->f_mode & FMODE_WRITE))
+		return 0;
+
+	return xillybus_myflush(filp->private_data, HZ); /* 1 second timeout */
+}
+
+static void xillybus_autoflush(struct work_struct *work)
+{
+	struct delayed_work *workitem = container_of(
+		work, struct delayed_work, work);
+	struct xilly_channel *channel = container_of(
+		workitem, struct xilly_channel, rd_workitem);
+	int rc;
+
+	rc = xillybus_myflush(channel, -1);
+	if (rc == -EINTR)
+		dev_warn(channel->endpoint->dev,
+			 "Autoflush failed because work queue thread got a signal.\n");
+	else if (rc)
+		dev_err(channel->endpoint->dev,
+			"Autoflush failed under weird circumstances.\n");
+}
+
+static ssize_t xillybus_write(struct file *filp, const char __user *userbuf,
+			      size_t count, loff_t *f_pos)
+{
+	ssize_t rc;
+	unsigned long flags;
+	int bytes_done = 0;
+	struct xilly_channel *channel = filp->private_data;
+
+	int full, exhausted;
+	/* Initializations are there only to silence warnings */
+
+	int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;
+	int end_offset_plus1 = 0;
+
+	if (channel->endpoint->fatal_error)
+		return -EIO;
+
+	rc = mutex_lock_interruptible(&channel->rd_mutex);
+	if (rc)
+		return rc;
+
+	while (1) {
+		int bytes_to_do = count - bytes_done;
+
+		spin_lock_irqsave(&channel->rd_spinlock, flags);
+
+		full = channel->rd_full;
+
+		if (!full) {
+			bufidx = channel->rd_host_buf_idx;
+			bufpos = channel->rd_host_buf_pos;
+			howmany = channel->rd_buf_size - bufpos;
+
+			/*
+			 * Update rd_host_* to its state after this operation.
+			 * count=0 means committing the buffer immediately,
+			 * which is like flushing, but not necessarily block.
+			 */
+
+			if ((howmany > bytes_to_do) &&
+			    (count ||
+			     ((bufpos >> channel->log2_element_size) == 0))) {
+				bufferdone = 0;
+
+				howmany = bytes_to_do;
+				channel->rd_host_buf_pos += howmany;
+			} else {
+				bufferdone = 1;
+
+				if (count) {
+					end_offset_plus1 =
+						channel->rd_buf_size >>
+						channel->log2_element_size;
+					channel->rd_host_buf_pos = 0;
+				} else {
+					unsigned char *tail;
+					int i;
+
+					howmany = 0;
+
+					end_offset_plus1 = bufpos >>
+						channel->log2_element_size;
+
+					channel->rd_host_buf_pos -=
+						end_offset_plus1 <<
+						channel->log2_element_size;
+
+					tail = channel->
+						rd_buffers[bufidx]->addr +
+						(end_offset_plus1 <<
+						 channel->log2_element_size);
+
+					for (i = 0;
+					     i < channel->rd_host_buf_pos;
+					     i++)
+						channel->rd_leftovers[i] =
+							*tail++;
+				}
+
+				if (bufidx == channel->rd_fpga_buf_idx)
+					channel->rd_full = 1;
+
+				if (bufidx >= (channel->num_rd_buffers - 1))
+					channel->rd_host_buf_idx = 0;
+				else
+					channel->rd_host_buf_idx++;
+			}
+		}
+
+		/*
+		 * Marking our situation after the possible changes above,
+		 * for use  after releasing the spinlock.
+		 *
+		 * full = full before change
+		 * exhasted = full after possible change
+		 */
+
+		exhausted = channel->rd_full;
+
+		spin_unlock_irqrestore(&channel->rd_spinlock, flags);
+
+		if (!full) { /* Go on, now without the spinlock */
+			unsigned char *head =
+				channel->rd_buffers[bufidx]->addr;
+			int i;
+
+			if ((bufpos == 0) || /* Zero means it's virgin */
+			    (channel->rd_leftovers[3] != 0)) {
+				dma_sync_single_for_cpu(channel->endpoint->dev,
+							channel->rd_buffers[bufidx]->dma_addr,
+							channel->rd_buf_size,
+							DMA_TO_DEVICE);
+
+				/* Virgin, but leftovers are due */
+				for (i = 0; i < bufpos; i++)
+					*head++ = channel->rd_leftovers[i];
+
+				channel->rd_leftovers[3] = 0; /* Clear flag */
+			}
+
+			if (copy_from_user(
+				    channel->rd_buffers[bufidx]->addr + bufpos,
+				    userbuf, howmany))
+				rc = -EFAULT;
+
+			userbuf += howmany;
+			bytes_done += howmany;
+
+			if (bufferdone) {
+				dma_sync_single_for_device(channel->endpoint->dev,
+							   channel->rd_buffers[bufidx]->dma_addr,
+							   channel->rd_buf_size,
+							   DMA_TO_DEVICE);
+
+				mutex_lock(&channel->endpoint->register_mutex);
+
+				iowrite32(end_offset_plus1 - 1,
+					  channel->endpoint->registers +
+					  fpga_buf_offset_reg);
+
+				iowrite32((channel->chan_num << 1) |
+					  (2 << 24) |  /* 2 = submit buffer */
+					  (bufidx << 12),
+					  channel->endpoint->registers +
+					  fpga_buf_ctrl_reg);
+
+				mutex_unlock(&channel->endpoint->
+					     register_mutex);
+
+				channel->rd_leftovers[3] =
+					(channel->rd_host_buf_pos != 0);
+			}
+
+			if (rc) {
+				mutex_unlock(&channel->rd_mutex);
+
+				if (channel->endpoint->fatal_error)
+					return -EIO;
+
+				if (!channel->rd_synchronous)
+					queue_delayed_work(
+						xillybus_wq,
+						&channel->rd_workitem,
+						XILLY_RX_TIMEOUT);
+
+				return rc;
+			}
+		}
+
+		if (bytes_done >= count)
+			break;
+
+		if (!exhausted)
+			continue; /* If there's more space, just go on */
+
+		if ((bytes_done > 0) && channel->rd_allow_partial)
+			break;
+
+		/*
+		 * Indefinite sleep with mutex taken. With data waiting for
+		 * flushing, user should not be surprised if open() for write
+		 * sleeps.
+		 */
+
+		if (filp->f_flags & O_NONBLOCK) {
+			rc = -EAGAIN;
+			break;
+		}
+
+		if (wait_event_interruptible(channel->rd_wait,
+					     (!channel->rd_full))) {
+			mutex_unlock(&channel->rd_mutex);
+
+			if (channel->endpoint->fatal_error)
+				return -EIO;
+
+			if (bytes_done)
+				return bytes_done;
+			return -EINTR;
+		}
+	}
+
+	mutex_unlock(&channel->rd_mutex);
+
+	if (!channel->rd_synchronous)
+		queue_delayed_work(xillybus_wq,
+				   &channel->rd_workitem,
+				   XILLY_RX_TIMEOUT);
+
+	if (channel->endpoint->fatal_error)
+		return -EIO;
+
+	if (rc)
+		return rc;
+
+	if ((channel->rd_synchronous) && (bytes_done > 0)) {
+		rc = xillybus_myflush(filp->private_data, 0); /* No timeout */
+
+		if (rc && (rc != -EINTR))
+			return rc;
+	}
+
+	return bytes_done;
+}
+
+static int xillybus_open(struct inode *inode, struct file *filp)
+{
+	int rc;
+	unsigned long flags;
+	struct xilly_endpoint *endpoint;
+	struct xilly_channel *channel;
+	int index;
+
+	rc = xillybus_find_inode(inode, (void **)&endpoint, &index);
+	if (rc)
+		return rc;
+
+	if (endpoint->fatal_error)
+		return -EIO;
+
+	channel = endpoint->channels[1 + index];
+	filp->private_data = channel;
+
+	/*
+	 * It gets complicated because:
+	 * 1. We don't want to take a mutex we don't have to
+	 * 2. We don't want to open one direction if the other will fail.
+	 */
+
+	if ((filp->f_mode & FMODE_READ) && (!channel->num_wr_buffers))
+		return -ENODEV;
+
+	if ((filp->f_mode & FMODE_WRITE) && (!channel->num_rd_buffers))
+		return -ENODEV;
+
+	if ((filp->f_mode & FMODE_READ) && (filp->f_flags & O_NONBLOCK) &&
+	    (channel->wr_synchronous || !channel->wr_allow_partial ||
+	     !channel->wr_supports_nonempty)) {
+		dev_err(endpoint->dev,
+			"open() failed: O_NONBLOCK not allowed for read on this device\n");
+		return -ENODEV;
+	}
+
+	if ((filp->f_mode & FMODE_WRITE) && (filp->f_flags & O_NONBLOCK) &&
+	    (channel->rd_synchronous || !channel->rd_allow_partial)) {
+		dev_err(endpoint->dev,
+			"open() failed: O_NONBLOCK not allowed for write on this device\n");
+		return -ENODEV;
+	}
+
+	/*
+	 * Note: open() may block on getting mutexes despite O_NONBLOCK.
+	 * This shouldn't occur normally, since multiple open of the same
+	 * file descriptor is almost always prohibited anyhow
+	 * (*_exclusive_open is normally set in real-life systems).
+	 */
+
+	if (filp->f_mode & FMODE_READ) {
+		rc = mutex_lock_interruptible(&channel->wr_mutex);
+		if (rc)
+			return rc;
+	}
+
+	if (filp->f_mode & FMODE_WRITE) {
+		rc = mutex_lock_interruptible(&channel->rd_mutex);
+		if (rc)
+			goto unlock_wr;
+	}
+
+	if ((filp->f_mode & FMODE_READ) &&
+	    (channel->wr_ref_count != 0) &&
+	    (channel->wr_exclusive_open)) {
+		rc = -EBUSY;
+		goto unlock;
+	}
+
+	if ((filp->f_mode & FMODE_WRITE) &&
+	    (channel->rd_ref_count != 0) &&
+	    (channel->rd_exclusive_open)) {
+		rc = -EBUSY;
+		goto unlock;
+	}
+
+	if (filp->f_mode & FMODE_READ) {
+		if (channel->wr_ref_count == 0) { /* First open of file */
+			/* Move the host to first buffer */
+			spin_lock_irqsave(&channel->wr_spinlock, flags);
+			channel->wr_host_buf_idx = 0;
+			channel->wr_host_buf_pos = 0;
+			channel->wr_fpga_buf_idx = -1;
+			channel->wr_empty = 1;
+			channel->wr_ready = 0;
+			channel->wr_sleepy = 1;
+			channel->wr_eof = -1;
+			channel->wr_hangup = 0;
+
+			spin_unlock_irqrestore(&channel->wr_spinlock, flags);
+
+			iowrite32(1 | (channel->chan_num << 1) |
+				  (4 << 24) |  /* Opcode 4, open channel */
+				  ((channel->wr_synchronous & 1) << 23),
+				  channel->endpoint->registers +
+				  fpga_buf_ctrl_reg);
+		}
+
+		channel->wr_ref_count++;
+	}
+
+	if (filp->f_mode & FMODE_WRITE) {
+		if (channel->rd_ref_count == 0) { /* First open of file */
+			/* Move the host to first buffer */
+			spin_lock_irqsave(&channel->rd_spinlock, flags);
+			channel->rd_host_buf_idx = 0;
+			channel->rd_host_buf_pos = 0;
+			channel->rd_leftovers[3] = 0; /* No leftovers. */
+			channel->rd_fpga_buf_idx = channel->num_rd_buffers - 1;
+			channel->rd_full = 0;
+
+			spin_unlock_irqrestore(&channel->rd_spinlock, flags);
+
+			iowrite32((channel->chan_num << 1) |
+				  (4 << 24),   /* Opcode 4, open channel */
+				  channel->endpoint->registers +
+				  fpga_buf_ctrl_reg);
+		}
+
+		channel->rd_ref_count++;
+	}
+
+unlock:
+	if (filp->f_mode & FMODE_WRITE)
+		mutex_unlock(&channel->rd_mutex);
+unlock_wr:
+	if (filp->f_mode & FMODE_READ)
+		mutex_unlock(&channel->wr_mutex);
+
+	if (!rc && (!channel->seekable))
+		return nonseekable_open(inode, filp);
+
+	return rc;
+}
+
+static int xillybus_release(struct inode *inode, struct file *filp)
+{
+	unsigned long flags;
+	struct xilly_channel *channel = filp->private_data;
+
+	int buf_idx;
+	int eof;
+
+	if (channel->endpoint->fatal_error)
+		return -EIO;
+
+	if (filp->f_mode & FMODE_WRITE) {
+		mutex_lock(&channel->rd_mutex);
+
+		channel->rd_ref_count--;
+
+		if (channel->rd_ref_count == 0) {
+			/*
+			 * We rely on the kernel calling flush()
+			 * before we get here.
+			 */
+
+			iowrite32((channel->chan_num << 1) | /* Channel ID */
+				  (5 << 24),  /* Opcode 5, close channel */
+				  channel->endpoint->registers +
+				  fpga_buf_ctrl_reg);
+		}
+		mutex_unlock(&channel->rd_mutex);
+	}
+
+	if (filp->f_mode & FMODE_READ) {
+		mutex_lock(&channel->wr_mutex);
+
+		channel->wr_ref_count--;
+
+		if (channel->wr_ref_count == 0) {
+			iowrite32(1 | (channel->chan_num << 1) |
+				  (5 << 24),  /* Opcode 5, close channel */
+				  channel->endpoint->registers +
+				  fpga_buf_ctrl_reg);
+
+			/*
+			 * This is crazily cautious: We make sure that not
+			 * only that we got an EOF (be it because we closed
+			 * the channel or because of a user's EOF), but verify
+			 * that it's one beyond the last buffer arrived, so
+			 * we have no leftover buffers pending before wrapping
+			 * up (which can only happen in asynchronous channels,
+			 * BTW)
+			 */
+
+			while (1) {
+				spin_lock_irqsave(&channel->wr_spinlock,
+						  flags);
+				buf_idx = channel->wr_fpga_buf_idx;
+				eof = channel->wr_eof;
+				channel->wr_sleepy = 1;
+				spin_unlock_irqrestore(&channel->wr_spinlock,
+						       flags);
+
+				/*
+				 * Check if eof points at the buffer after
+				 * the last one the FPGA submitted. Note that
+				 * no EOF is marked by negative eof.
+				 */
+
+				buf_idx++;
+				if (buf_idx == channel->num_wr_buffers)
+					buf_idx = 0;
+
+				if (buf_idx == eof)
+					break;
+
+				/*
+				 * Steal extra 100 ms if awaken by interrupt.
+				 * This is a simple workaround for an
+				 * interrupt pending when entering, which would
+				 * otherwise result in declaring the hardware
+				 * non-responsive.
+				 */
+
+				if (wait_event_interruptible(
+					    channel->wr_wait,
+					    (!channel->wr_sleepy)))
+					msleep(100);
+
+				if (channel->wr_sleepy) {
+					mutex_unlock(&channel->wr_mutex);
+					dev_warn(channel->endpoint->dev,
+						 "Hardware failed to respond to close command, therefore left in messy state.\n");
+					return -EINTR;
+				}
+			}
+		}
+
+		mutex_unlock(&channel->wr_mutex);
+	}
+
+	return 0;
+}
+
+static loff_t xillybus_llseek(struct file *filp, loff_t offset, int whence)
+{
+	struct xilly_channel *channel = filp->private_data;
+	loff_t pos = filp->f_pos;
+	int rc = 0;
+
+	/*
+	 * Take both mutexes not allowing interrupts, since it seems like
+	 * common applications don't expect an -EINTR here. Besides, multiple
+	 * access to a single file descriptor on seekable devices is a mess
+	 * anyhow.
+	 */
+
+	if (channel->endpoint->fatal_error)
+		return -EIO;
+
+	mutex_lock(&channel->wr_mutex);
+	mutex_lock(&channel->rd_mutex);
+
+	switch (whence) {
+	case SEEK_SET:
+		pos = offset;
+		break;
+	case SEEK_CUR:
+		pos += offset;
+		break;
+	case SEEK_END:
+		pos = offset; /* Going to the end => to the beginning */
+		break;
+	default:
+		rc = -EINVAL;
+		goto end;
+	}
+
+	/* In any case, we must finish on an element boundary */
+	if (pos & ((1 << channel->log2_element_size) - 1)) {
+		rc = -EINVAL;
+		goto end;
+	}
+
+	mutex_lock(&channel->endpoint->register_mutex);
+
+	iowrite32(pos >> channel->log2_element_size,
+		  channel->endpoint->registers + fpga_buf_offset_reg);
+
+	iowrite32((channel->chan_num << 1) |
+		  (6 << 24),  /* Opcode 6, set address */
+		  channel->endpoint->registers + fpga_buf_ctrl_reg);
+
+	mutex_unlock(&channel->endpoint->register_mutex);
+
+end:
+	mutex_unlock(&channel->rd_mutex);
+	mutex_unlock(&channel->wr_mutex);
+
+	if (rc) /* Return error after releasing mutexes */
+		return rc;
+
+	filp->f_pos = pos;
+
+	/*
+	 * Since seekable devices are allowed only when the channel is
+	 * synchronous, we assume that there is no data pending in either
+	 * direction (which holds true as long as no concurrent access on the
+	 * file descriptor takes place).
+	 * The only thing we may need to throw away is leftovers from partial
+	 * write() flush.
+	 */
+
+	channel->rd_leftovers[3] = 0;
+
+	return pos;
+}
+
+static __poll_t xillybus_poll(struct file *filp, poll_table *wait)
+{
+	struct xilly_channel *channel = filp->private_data;
+	__poll_t mask = 0;
+	unsigned long flags;
+
+	poll_wait(filp, &channel->endpoint->ep_wait, wait);
+
+	/*
+	 * poll() won't play ball regarding read() channels which
+	 * aren't asynchronous and support the nonempty message. Allowing
+	 * that will create situations where data has been delivered at
+	 * the FPGA, and users expecting select() to wake up, which it may
+	 * not.
+	 */
+
+	if (!channel->wr_synchronous && channel->wr_supports_nonempty) {
+		poll_wait(filp, &channel->wr_wait, wait);
+		poll_wait(filp, &channel->wr_ready_wait, wait);
+
+		spin_lock_irqsave(&channel->wr_spinlock, flags);
+		if (!channel->wr_empty || channel->wr_ready)
+			mask |= EPOLLIN | EPOLLRDNORM;
+
+		if (channel->wr_hangup)
+			/*
+			 * Not EPOLLHUP, because its behavior is in the
+			 * mist, and EPOLLIN does what we want: Wake up
+			 * the read file descriptor so it sees EOF.
+			 */
+			mask |=  EPOLLIN | EPOLLRDNORM;
+		spin_unlock_irqrestore(&channel->wr_spinlock, flags);
+	}
+
+	/*
+	 * If partial data write is disallowed on a write() channel,
+	 * it's pointless to ever signal OK to write, because is could
+	 * block despite some space being available.
+	 */
+
+	if (channel->rd_allow_partial) {
+		poll_wait(filp, &channel->rd_wait, wait);
+
+		spin_lock_irqsave(&channel->rd_spinlock, flags);
+		if (!channel->rd_full)
+			mask |= EPOLLOUT | EPOLLWRNORM;
+		spin_unlock_irqrestore(&channel->rd_spinlock, flags);
+	}
+
+	if (channel->endpoint->fatal_error)
+		mask |= EPOLLERR;
+
+	return mask;
+}
+
+static const struct file_operations xillybus_fops = {
+	.owner      = THIS_MODULE,
+	.read       = xillybus_read,
+	.write      = xillybus_write,
+	.open       = xillybus_open,
+	.flush      = xillybus_flush,
+	.release    = xillybus_release,
+	.llseek     = xillybus_llseek,
+	.poll       = xillybus_poll,
+};
+
+struct xilly_endpoint *xillybus_init_endpoint(struct device *dev)
+{
+	struct xilly_endpoint *endpoint;
+
+	endpoint = devm_kzalloc(dev, sizeof(*endpoint), GFP_KERNEL);
+	if (!endpoint)
+		return NULL;
+
+	endpoint->dev = dev;
+	endpoint->msg_counter = 0x0b;
+	endpoint->failed_messages = 0;
+	endpoint->fatal_error = 0;
+
+	init_waitqueue_head(&endpoint->ep_wait);
+	mutex_init(&endpoint->register_mutex);
+
+	return endpoint;
+}
+EXPORT_SYMBOL(xillybus_init_endpoint);
+
+static int xilly_quiesce(struct xilly_endpoint *endpoint)
+{
+	long t;
+
+	endpoint->idtlen = -1;
+
+	iowrite32((u32) (endpoint->dma_using_dac & 0x0001),
+		  endpoint->registers + fpga_dma_control_reg);
+
+	t = wait_event_interruptible_timeout(endpoint->ep_wait,
+					     (endpoint->idtlen >= 0),
+					     XILLY_TIMEOUT);
+	if (t <= 0) {
+		dev_err(endpoint->dev,
+			"Failed to quiesce the device on exit.\n");
+		return -ENODEV;
+	}
+	return 0;
+}
+
+int xillybus_endpoint_discovery(struct xilly_endpoint *endpoint)
+{
+	int rc;
+	long t;
+
+	void *bootstrap_resources;
+	int idtbuffersize = (1 << PAGE_SHIFT);
+	struct device *dev = endpoint->dev;
+
+	/*
+	 * The bogus IDT is used during bootstrap for allocating the initial
+	 * message buffer, and then the message buffer and space for the IDT
+	 * itself. The initial message buffer is of a single page's size, but
+	 * it's soon replaced with a more modest one (and memory is freed).
+	 */
+
+	unsigned char bogus_idt[8] = { 1, 224, (PAGE_SHIFT)-2, 0,
+				       3, 192, PAGE_SHIFT, 0 };
+	struct xilly_idt_handle idt_handle;
+
+	/*
+	 * Writing the value 0x00000001 to Endianness register signals which
+	 * endianness this processor is using, so the FPGA can swap words as
+	 * necessary.
+	 */
+
+	iowrite32(1, endpoint->registers + fpga_endian_reg);
+
+	/* Bootstrap phase I: Allocate temporary message buffer */
+
+	bootstrap_resources = devres_open_group(dev, NULL, GFP_KERNEL);
+	if (!bootstrap_resources)
+		return -ENOMEM;
+
+	endpoint->num_channels = 0;
+
+	rc = xilly_setupchannels(endpoint, bogus_idt, 1);
+	if (rc)
+		return rc;
+
+	/* Clear the message subsystem (and counter in particular) */
+	iowrite32(0x04, endpoint->registers + fpga_msg_ctrl_reg);
+
+	endpoint->idtlen = -1;
+
+	/*
+	 * Set DMA 32/64 bit mode, quiesce the device (?!) and get IDT
+	 * buffer size.
+	 */
+	iowrite32((u32) (endpoint->dma_using_dac & 0x0001),
+		  endpoint->registers + fpga_dma_control_reg);
+
+	t = wait_event_interruptible_timeout(endpoint->ep_wait,
+					     (endpoint->idtlen >= 0),
+					     XILLY_TIMEOUT);
+	if (t <= 0) {
+		dev_err(endpoint->dev, "No response from FPGA. Aborting.\n");
+		return -ENODEV;
+	}
+
+	/* Enable DMA */
+	iowrite32((u32) (0x0002 | (endpoint->dma_using_dac & 0x0001)),
+		  endpoint->registers + fpga_dma_control_reg);
+
+	/* Bootstrap phase II: Allocate buffer for IDT and obtain it */
+	while (endpoint->idtlen >= idtbuffersize) {
+		idtbuffersize *= 2;
+		bogus_idt[6]++;
+	}
+
+	endpoint->num_channels = 1;
+
+	rc = xilly_setupchannels(endpoint, bogus_idt, 2);
+	if (rc)
+		goto failed_idt;
+
+	rc = xilly_obtain_idt(endpoint);
+	if (rc)
+		goto failed_idt;
+
+	rc = xilly_scan_idt(endpoint, &idt_handle);
+	if (rc)
+		goto failed_idt;
+
+	devres_close_group(dev, bootstrap_resources);
+
+	/* Bootstrap phase III: Allocate buffers according to IDT */
+
+	rc = xilly_setupchannels(endpoint,
+				 idt_handle.chandesc,
+				 idt_handle.entries);
+	if (rc)
+		goto failed_idt;
+
+	rc = xillybus_init_chrdev(dev, &xillybus_fops,
+				  endpoint->owner, endpoint,
+				  idt_handle.names,
+				  idt_handle.names_len,
+				  endpoint->num_channels,
+				  xillyname, false);
+
+	if (rc)
+		goto failed_idt;
+
+	devres_release_group(dev, bootstrap_resources);
+
+	return 0;
+
+failed_idt:
+	xilly_quiesce(endpoint);
+	flush_workqueue(xillybus_wq);
+
+	return rc;
+}
+EXPORT_SYMBOL(xillybus_endpoint_discovery);
+
+void xillybus_endpoint_remove(struct xilly_endpoint *endpoint)
+{
+	xillybus_cleanup_chrdev(endpoint, endpoint->dev);
+
+	xilly_quiesce(endpoint);
+
+	/*
+	 * Flushing is done upon endpoint release to prevent access to memory
+	 * just about to be released. This makes the quiesce complete.
+	 */
+	flush_workqueue(xillybus_wq);
+}
+EXPORT_SYMBOL(xillybus_endpoint_remove);
+
+static int __init xillybus_init(void)
+{
+	xillybus_wq = alloc_workqueue(xillyname, 0, 0);
+	if (!xillybus_wq)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void __exit xillybus_exit(void)
+{
+	/* flush_workqueue() was called for each endpoint released */
+	destroy_workqueue(xillybus_wq);
+}
+
+module_init(xillybus_init);
+module_exit(xillybus_exit);
diff --git a/drivers/char/xillybus/xillybus_of.c b/drivers/char/xillybus/xillybus_of.c
new file mode 100644
index 000000000..e5372e45d
--- /dev/null
+++ b/drivers/char/xillybus/xillybus_of.c
@@ -0,0 +1,86 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * linux/drivers/misc/xillybus_of.c
+ *
+ * Copyright 2011 Xillybus Ltd, http://xillybus.com
+ *
+ * Driver for the Xillybus FPGA/host framework using Open Firmware.
+ */
+
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/slab.h>
+#include <linux/platform_device.h>
+#include <linux/of.h>
+#include <linux/err.h>
+#include "xillybus.h"
+
+MODULE_DESCRIPTION("Xillybus driver for Open Firmware");
+MODULE_AUTHOR("Eli Billauer, Xillybus Ltd.");
+MODULE_ALIAS("xillybus_of");
+MODULE_LICENSE("GPL v2");
+
+static const char xillyname[] = "xillybus_of";
+
+/* Match table for of_platform binding */
+static const struct of_device_id xillybus_of_match[] = {
+	{ .compatible = "xillybus,xillybus-1.00.a", },
+	{ .compatible = "xlnx,xillybus-1.00.a", }, /* Deprecated */
+	{}
+};
+
+MODULE_DEVICE_TABLE(of, xillybus_of_match);
+
+static int xilly_drv_probe(struct platform_device *op)
+{
+	struct device *dev = &op->dev;
+	struct xilly_endpoint *endpoint;
+	int rc;
+	int irq;
+
+	endpoint = xillybus_init_endpoint(dev);
+
+	if (!endpoint)
+		return -ENOMEM;
+
+	dev_set_drvdata(dev, endpoint);
+
+	endpoint->owner = THIS_MODULE;
+
+	endpoint->registers = devm_platform_ioremap_resource(op, 0);
+	if (IS_ERR(endpoint->registers))
+		return PTR_ERR(endpoint->registers);
+
+	irq = platform_get_irq(op, 0);
+
+	rc = devm_request_irq(dev, irq, xillybus_isr, 0, xillyname, endpoint);
+
+	if (rc) {
+		dev_err(endpoint->dev,
+			"Failed to register IRQ handler. Aborting.\n");
+		return -ENODEV;
+	}
+
+	return xillybus_endpoint_discovery(endpoint);
+}
+
+static int xilly_drv_remove(struct platform_device *op)
+{
+	struct device *dev = &op->dev;
+	struct xilly_endpoint *endpoint = dev_get_drvdata(dev);
+
+	xillybus_endpoint_remove(endpoint);
+
+	return 0;
+}
+
+static struct platform_driver xillybus_platform_driver = {
+	.probe = xilly_drv_probe,
+	.remove = xilly_drv_remove,
+	.driver = {
+		.name = xillyname,
+		.of_match_table = xillybus_of_match,
+	},
+};
+
+module_platform_driver(xillybus_platform_driver);
diff --git a/drivers/char/xillybus/xillybus_pcie.c b/drivers/char/xillybus/xillybus_pcie.c
new file mode 100644
index 000000000..9858711e3
--- /dev/null
+++ b/drivers/char/xillybus/xillybus_pcie.c
@@ -0,0 +1,127 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * linux/drivers/misc/xillybus_pcie.c
+ *
+ * Copyright 2011 Xillybus Ltd, http://xillybus.com
+ *
+ * Driver for the Xillybus FPGA/host framework using PCI Express.
+ */
+
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+#include "xillybus.h"
+
+MODULE_DESCRIPTION("Xillybus driver for PCIe");
+MODULE_AUTHOR("Eli Billauer, Xillybus Ltd.");
+MODULE_ALIAS("xillybus_pcie");
+MODULE_LICENSE("GPL v2");
+
+#define PCI_DEVICE_ID_XILLYBUS		0xebeb
+
+#define PCI_VENDOR_ID_ACTEL		0x11aa
+#define PCI_VENDOR_ID_LATTICE		0x1204
+
+static const char xillyname[] = "xillybus_pcie";
+
+static const struct pci_device_id xillyids[] = {
+	{PCI_DEVICE(PCI_VENDOR_ID_XILINX, PCI_DEVICE_ID_XILLYBUS)},
+	{PCI_DEVICE(PCI_VENDOR_ID_ALTERA, PCI_DEVICE_ID_XILLYBUS)},
+	{PCI_DEVICE(PCI_VENDOR_ID_ACTEL, PCI_DEVICE_ID_XILLYBUS)},
+	{PCI_DEVICE(PCI_VENDOR_ID_LATTICE, PCI_DEVICE_ID_XILLYBUS)},
+	{ /* End: all zeroes */ }
+};
+
+static int xilly_probe(struct pci_dev *pdev,
+		       const struct pci_device_id *ent)
+{
+	struct xilly_endpoint *endpoint;
+	int rc;
+
+	endpoint = xillybus_init_endpoint(&pdev->dev);
+
+	if (!endpoint)
+		return -ENOMEM;
+
+	pci_set_drvdata(pdev, endpoint);
+
+	endpoint->owner = THIS_MODULE;
+
+	rc = pcim_enable_device(pdev);
+	if (rc) {
+		dev_err(endpoint->dev,
+			"pcim_enable_device() failed. Aborting.\n");
+		return rc;
+	}
+
+	/* L0s has caused packet drops. No power saving, thank you. */
+
+	pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S);
+
+	if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
+		dev_err(endpoint->dev,
+			"Incorrect BAR configuration. Aborting.\n");
+		return -ENODEV;
+	}
+
+	rc = pcim_iomap_regions(pdev, 0x01, xillyname);
+	if (rc) {
+		dev_err(endpoint->dev,
+			"pcim_iomap_regions() failed. Aborting.\n");
+		return rc;
+	}
+
+	endpoint->registers = pcim_iomap_table(pdev)[0];
+
+	pci_set_master(pdev);
+
+	/* Set up a single MSI interrupt */
+	if (pci_enable_msi(pdev)) {
+		dev_err(endpoint->dev,
+			"Failed to enable MSI interrupts. Aborting.\n");
+		return -ENODEV;
+	}
+	rc = devm_request_irq(&pdev->dev, pdev->irq, xillybus_isr, 0,
+			      xillyname, endpoint);
+	if (rc) {
+		dev_err(endpoint->dev,
+			"Failed to register MSI handler. Aborting.\n");
+		return -ENODEV;
+	}
+
+	/*
+	 * Some (old and buggy?) hardware drops 64-bit addressed PCIe packets,
+	 * even when the PCIe driver claims that a 64-bit mask is OK. On the
+	 * other hand, on some architectures, 64-bit addressing is mandatory.
+	 * So go for the 64-bit mask only when failing is the other option.
+	 */
+
+	if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(32))) {
+		endpoint->dma_using_dac = 0;
+	} else if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64))) {
+		endpoint->dma_using_dac = 1;
+	} else {
+		dev_err(endpoint->dev, "Failed to set DMA mask. Aborting.\n");
+		return -ENODEV;
+	}
+
+	return xillybus_endpoint_discovery(endpoint);
+}
+
+static void xilly_remove(struct pci_dev *pdev)
+{
+	struct xilly_endpoint *endpoint = pci_get_drvdata(pdev);
+
+	xillybus_endpoint_remove(endpoint);
+}
+
+MODULE_DEVICE_TABLE(pci, xillyids);
+
+static struct pci_driver xillybus_driver = {
+	.name = xillyname,
+	.id_table = xillyids,
+	.probe = xilly_probe,
+	.remove = xilly_remove,
+};
+
+module_pci_driver(xillybus_driver);
diff --git a/drivers/char/xillybus/xillyusb.c b/drivers/char/xillybus/xillyusb.c
new file mode 100644
index 000000000..39bcbfd90
--- /dev/null
+++ b/drivers/char/xillybus/xillyusb.c
@@ -0,0 +1,2261 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright 2020 Xillybus Ltd, http://xillybus.com
+ *
+ * Driver for the XillyUSB FPGA/host framework.
+ *
+ * This driver interfaces with a special IP core in an FPGA, setting up
+ * a pipe between a hardware FIFO in the programmable logic and a device
+ * file in the host. The number of such pipes and their attributes are
+ * set up on the logic. This driver detects these automatically and
+ * creates the device files accordingly.
+ */
+
+#include <linux/types.h>
+#include <linux/slab.h>
+#include <linux/list.h>
+#include <linux/device.h>
+#include <linux/module.h>
+#include <asm/byteorder.h>
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/sched.h>
+#include <linux/fs.h>
+#include <linux/spinlock.h>
+#include <linux/mutex.h>
+#include <linux/workqueue.h>
+#include <linux/crc32.h>
+#include <linux/poll.h>
+#include <linux/delay.h>
+#include <linux/usb.h>
+
+#include "xillybus_class.h"
+
+MODULE_DESCRIPTION("Driver for XillyUSB FPGA IP Core");
+MODULE_AUTHOR("Eli Billauer, Xillybus Ltd.");
+MODULE_ALIAS("xillyusb");
+MODULE_LICENSE("GPL v2");
+
+#define XILLY_RX_TIMEOUT		(10 * HZ / 1000)
+#define XILLY_RESPONSE_TIMEOUT		(500 * HZ / 1000)
+
+#define BUF_SIZE_ORDER			4
+#define BUFNUM				8
+#define LOG2_IDT_FIFO_SIZE		16
+#define LOG2_INITIAL_FIFO_BUF_SIZE	16
+
+#define MSG_EP_NUM			1
+#define IN_EP_NUM			1
+
+static const char xillyname[] = "xillyusb";
+
+static unsigned int fifo_buf_order;
+
+#define USB_VENDOR_ID_XILINX		0x03fd
+#define USB_VENDOR_ID_ALTERA		0x09fb
+
+#define USB_PRODUCT_ID_XILLYUSB		0xebbe
+
+static const struct usb_device_id xillyusb_table[] = {
+	{ USB_DEVICE(USB_VENDOR_ID_XILINX, USB_PRODUCT_ID_XILLYUSB) },
+	{ USB_DEVICE(USB_VENDOR_ID_ALTERA, USB_PRODUCT_ID_XILLYUSB) },
+	{ }
+};
+
+MODULE_DEVICE_TABLE(usb, xillyusb_table);
+
+struct xillyusb_dev;
+
+struct xillyfifo {
+	unsigned int bufsize; /* In bytes, always a power of 2 */
+	unsigned int bufnum;
+	unsigned int size; /* Lazy: Equals bufsize * bufnum */
+	unsigned int buf_order;
+
+	int fill; /* Number of bytes in the FIFO */
+	spinlock_t lock;
+	wait_queue_head_t waitq;
+
+	unsigned int readpos;
+	unsigned int readbuf;
+	unsigned int writepos;
+	unsigned int writebuf;
+	char **mem;
+};
+
+struct xillyusb_channel;
+
+struct xillyusb_endpoint {
+	struct xillyusb_dev *xdev;
+
+	struct mutex ep_mutex; /* serialize operations on endpoint */
+
+	struct list_head buffers;
+	struct list_head filled_buffers;
+	spinlock_t buffers_lock; /* protect these two lists */
+
+	unsigned int order;
+	unsigned int buffer_size;
+
+	unsigned int fill_mask;
+
+	int outstanding_urbs;
+
+	struct usb_anchor anchor;
+
+	struct xillyfifo fifo;
+
+	struct work_struct workitem;
+
+	bool shutting_down;
+	bool drained;
+	bool wake_on_drain;
+
+	u8 ep_num;
+};
+
+struct xillyusb_channel {
+	struct xillyusb_dev *xdev;
+
+	struct xillyfifo *in_fifo;
+	struct xillyusb_endpoint *out_ep;
+	struct mutex lock; /* protect @out_ep, @in_fifo, bit fields below */
+
+	struct mutex in_mutex; /* serialize fops on FPGA to host stream */
+	struct mutex out_mutex; /* serialize fops on host to FPGA stream */
+	wait_queue_head_t flushq;
+
+	int chan_idx;
+
+	u32 in_consumed_bytes;
+	u32 in_current_checkpoint;
+	u32 out_bytes;
+
+	unsigned int in_log2_element_size;
+	unsigned int out_log2_element_size;
+	unsigned int in_log2_fifo_size;
+	unsigned int out_log2_fifo_size;
+
+	unsigned int read_data_ok; /* EOF not arrived (yet) */
+	unsigned int poll_used;
+	unsigned int flushing;
+	unsigned int flushed;
+	unsigned int canceled;
+
+	/* Bit fields protected by @lock except for initialization */
+	unsigned readable:1;
+	unsigned writable:1;
+	unsigned open_for_read:1;
+	unsigned open_for_write:1;
+	unsigned in_synchronous:1;
+	unsigned out_synchronous:1;
+	unsigned in_seekable:1;
+	unsigned out_seekable:1;
+};
+
+struct xillybuffer {
+	struct list_head entry;
+	struct xillyusb_endpoint *ep;
+	void *buf;
+	unsigned int len;
+};
+
+struct xillyusb_dev {
+	struct xillyusb_channel *channels;
+
+	struct usb_device	*udev;
+	struct device		*dev; /* For dev_err() and such */
+	struct kref		kref;
+	struct workqueue_struct	*workq;
+
+	int error;
+	spinlock_t error_lock; /* protect @error */
+	struct work_struct wakeup_workitem;
+
+	int num_channels;
+
+	struct xillyusb_endpoint *msg_ep;
+	struct xillyusb_endpoint *in_ep;
+
+	struct mutex msg_mutex; /* serialize opcode transmission */
+	int in_bytes_left;
+	int leftover_chan_num;
+	unsigned int in_counter;
+	struct mutex process_in_mutex; /* synchronize wakeup_all() */
+};
+
+/* FPGA to host opcodes */
+enum {
+	OPCODE_DATA = 0,
+	OPCODE_QUIESCE_ACK = 1,
+	OPCODE_EOF = 2,
+	OPCODE_REACHED_CHECKPOINT = 3,
+	OPCODE_CANCELED_CHECKPOINT = 4,
+};
+
+/* Host to FPGA opcodes */
+enum {
+	OPCODE_QUIESCE = 0,
+	OPCODE_REQ_IDT = 1,
+	OPCODE_SET_CHECKPOINT = 2,
+	OPCODE_CLOSE = 3,
+	OPCODE_SET_PUSH = 4,
+	OPCODE_UPDATE_PUSH = 5,
+	OPCODE_CANCEL_CHECKPOINT = 6,
+	OPCODE_SET_ADDR = 7,
+};
+
+/*
+ * fifo_write() and fifo_read() are NOT reentrant (i.e. concurrent multiple
+ * calls to each on the same FIFO is not allowed) however it's OK to have
+ * threads calling each of the two functions once on the same FIFO, and
+ * at the same time.
+ */
+
+static int fifo_write(struct xillyfifo *fifo,
+		      const void *data, unsigned int len,
+		      int (*copier)(void *, const void *, int))
+{
+	unsigned int done = 0;
+	unsigned int todo = len;
+	unsigned int nmax;
+	unsigned int writepos = fifo->writepos;
+	unsigned int writebuf = fifo->writebuf;
+	unsigned long flags;
+	int rc;
+
+	nmax = fifo->size - READ_ONCE(fifo->fill);
+
+	while (1) {
+		unsigned int nrail = fifo->bufsize - writepos;
+		unsigned int n = min(todo, nmax);
+
+		if (n == 0) {
+			spin_lock_irqsave(&fifo->lock, flags);
+			fifo->fill += done;
+			spin_unlock_irqrestore(&fifo->lock, flags);
+
+			fifo->writepos = writepos;
+			fifo->writebuf = writebuf;
+
+			return done;
+		}
+
+		if (n > nrail)
+			n = nrail;
+
+		rc = (*copier)(fifo->mem[writebuf] + writepos, data + done, n);
+
+		if (rc)
+			return rc;
+
+		done += n;
+		todo -= n;
+
+		writepos += n;
+		nmax -= n;
+
+		if (writepos == fifo->bufsize) {
+			writepos = 0;
+			writebuf++;
+
+			if (writebuf == fifo->bufnum)
+				writebuf = 0;
+		}
+	}
+}
+
+static int fifo_read(struct xillyfifo *fifo,
+		     void *data, unsigned int len,
+		     int (*copier)(void *, const void *, int))
+{
+	unsigned int done = 0;
+	unsigned int todo = len;
+	unsigned int fill;
+	unsigned int readpos = fifo->readpos;
+	unsigned int readbuf = fifo->readbuf;
+	unsigned long flags;
+	int rc;
+
+	/*
+	 * The spinlock here is necessary, because otherwise fifo->fill
+	 * could have been increased by fifo_write() after writing data
+	 * to the buffer, but this data would potentially not have been
+	 * visible on this thread at the time the updated fifo->fill was.
+	 * That could lead to reading invalid data.
+	 */
+
+	spin_lock_irqsave(&fifo->lock, flags);
+	fill = fifo->fill;
+	spin_unlock_irqrestore(&fifo->lock, flags);
+
+	while (1) {
+		unsigned int nrail = fifo->bufsize - readpos;
+		unsigned int n = min(todo, fill);
+
+		if (n == 0) {
+			spin_lock_irqsave(&fifo->lock, flags);
+			fifo->fill -= done;
+			spin_unlock_irqrestore(&fifo->lock, flags);
+
+			fifo->readpos = readpos;
+			fifo->readbuf = readbuf;
+
+			return done;
+		}
+
+		if (n > nrail)
+			n = nrail;
+
+		rc = (*copier)(data + done, fifo->mem[readbuf] + readpos, n);
+
+		if (rc)
+			return rc;
+
+		done += n;
+		todo -= n;
+
+		readpos += n;
+		fill -= n;
+
+		if (readpos == fifo->bufsize) {
+			readpos = 0;
+			readbuf++;
+
+			if (readbuf == fifo->bufnum)
+				readbuf = 0;
+		}
+	}
+}
+
+/*
+ * These three wrapper functions are used as the @copier argument to
+ * fifo_write() and fifo_read(), so that they can work directly with
+ * user memory as well.
+ */
+
+static int xilly_copy_from_user(void *dst, const void *src, int n)
+{
+	if (copy_from_user(dst, (const void __user *)src, n))
+		return -EFAULT;
+
+	return 0;
+}
+
+static int xilly_copy_to_user(void *dst, const void *src, int n)
+{
+	if (copy_to_user((void __user *)dst, src, n))
+		return -EFAULT;
+
+	return 0;
+}
+
+static int xilly_memcpy(void *dst, const void *src, int n)
+{
+	memcpy(dst, src, n);
+
+	return 0;
+}
+
+static int fifo_init(struct xillyfifo *fifo,
+		     unsigned int log2_size)
+{
+	unsigned int log2_bufnum;
+	unsigned int buf_order;
+	int i;
+
+	unsigned int log2_fifo_buf_size;
+
+retry:
+	log2_fifo_buf_size = fifo_buf_order + PAGE_SHIFT;
+
+	if (log2_size > log2_fifo_buf_size) {
+		log2_bufnum = log2_size - log2_fifo_buf_size;
+		buf_order = fifo_buf_order;
+		fifo->bufsize = 1 << log2_fifo_buf_size;
+	} else {
+		log2_bufnum = 0;
+		buf_order = (log2_size > PAGE_SHIFT) ?
+			log2_size - PAGE_SHIFT : 0;
+		fifo->bufsize = 1 << log2_size;
+	}
+
+	fifo->bufnum = 1 << log2_bufnum;
+	fifo->size = fifo->bufnum * fifo->bufsize;
+	fifo->buf_order = buf_order;
+
+	fifo->mem = kmalloc_array(fifo->bufnum, sizeof(void *), GFP_KERNEL);
+
+	if (!fifo->mem)
+		return -ENOMEM;
+
+	for (i = 0; i < fifo->bufnum; i++) {
+		fifo->mem[i] = (void *)
+			__get_free_pages(GFP_KERNEL, buf_order);
+
+		if (!fifo->mem[i])
+			goto memfail;
+	}
+
+	fifo->fill = 0;
+	fifo->readpos = 0;
+	fifo->readbuf = 0;
+	fifo->writepos = 0;
+	fifo->writebuf = 0;
+	spin_lock_init(&fifo->lock);
+	init_waitqueue_head(&fifo->waitq);
+	return 0;
+
+memfail:
+	for (i--; i >= 0; i--)
+		free_pages((unsigned long)fifo->mem[i], buf_order);
+
+	kfree(fifo->mem);
+	fifo->mem = NULL;
+
+	if (fifo_buf_order) {
+		fifo_buf_order--;
+		goto retry;
+	} else {
+		return -ENOMEM;
+	}
+}
+
+static void fifo_mem_release(struct xillyfifo *fifo)
+{
+	int i;
+
+	if (!fifo->mem)
+		return;
+
+	for (i = 0; i < fifo->bufnum; i++)
+		free_pages((unsigned long)fifo->mem[i], fifo->buf_order);
+
+	kfree(fifo->mem);
+}
+
+/*
+ * When endpoint_quiesce() returns, the endpoint has no URBs submitted,
+ * won't accept any new URB submissions, and its related work item doesn't
+ * and won't run anymore.
+ */
+
+static void endpoint_quiesce(struct xillyusb_endpoint *ep)
+{
+	mutex_lock(&ep->ep_mutex);
+	ep->shutting_down = true;
+	mutex_unlock(&ep->ep_mutex);
+
+	usb_kill_anchored_urbs(&ep->anchor);
+	cancel_work_sync(&ep->workitem);
+}
+
+/*
+ * Note that endpoint_dealloc() also frees fifo memory (if allocated), even
+ * though endpoint_alloc doesn't allocate that memory.
+ */
+
+static void endpoint_dealloc(struct xillyusb_endpoint *ep)
+{
+	struct list_head *this, *next;
+
+	fifo_mem_release(&ep->fifo);
+
+	/* Join @filled_buffers with @buffers to free these entries too */
+	list_splice(&ep->filled_buffers, &ep->buffers);
+
+	list_for_each_safe(this, next, &ep->buffers) {
+		struct xillybuffer *xb =
+			list_entry(this, struct xillybuffer, entry);
+
+		free_pages((unsigned long)xb->buf, ep->order);
+		kfree(xb);
+	}
+
+	kfree(ep);
+}
+
+static struct xillyusb_endpoint
+*endpoint_alloc(struct xillyusb_dev *xdev,
+		u8 ep_num,
+		void (*work)(struct work_struct *),
+		unsigned int order,
+		int bufnum)
+{
+	int i;
+
+	struct xillyusb_endpoint *ep;
+
+	ep = kzalloc(sizeof(*ep), GFP_KERNEL);
+
+	if (!ep)
+		return NULL;
+
+	INIT_LIST_HEAD(&ep->buffers);
+	INIT_LIST_HEAD(&ep->filled_buffers);
+
+	spin_lock_init(&ep->buffers_lock);
+	mutex_init(&ep->ep_mutex);
+
+	init_usb_anchor(&ep->anchor);
+	INIT_WORK(&ep->workitem, work);
+
+	ep->order = order;
+	ep->buffer_size =  1 << (PAGE_SHIFT + order);
+	ep->outstanding_urbs = 0;
+	ep->drained = true;
+	ep->wake_on_drain = false;
+	ep->xdev = xdev;
+	ep->ep_num = ep_num;
+	ep->shutting_down = false;
+
+	for (i = 0; i < bufnum; i++) {
+		struct xillybuffer *xb;
+		unsigned long addr;
+
+		xb = kzalloc(sizeof(*xb), GFP_KERNEL);
+
+		if (!xb) {
+			endpoint_dealloc(ep);
+			return NULL;
+		}
+
+		addr = __get_free_pages(GFP_KERNEL, order);
+
+		if (!addr) {
+			kfree(xb);
+			endpoint_dealloc(ep);
+			return NULL;
+		}
+
+		xb->buf = (void *)addr;
+		xb->ep = ep;
+		list_add_tail(&xb->entry, &ep->buffers);
+	}
+	return ep;
+}
+
+static void cleanup_dev(struct kref *kref)
+{
+	struct xillyusb_dev *xdev =
+		container_of(kref, struct xillyusb_dev, kref);
+
+	if (xdev->in_ep)
+		endpoint_dealloc(xdev->in_ep);
+
+	if (xdev->msg_ep)
+		endpoint_dealloc(xdev->msg_ep);
+
+	if (xdev->workq)
+		destroy_workqueue(xdev->workq);
+
+	usb_put_dev(xdev->udev);
+	kfree(xdev->channels); /* Argument may be NULL, and that's fine */
+	kfree(xdev);
+}
+
+/*
+ * @process_in_mutex is taken to ensure that bulk_in_work() won't call
+ * process_bulk_in() after wakeup_all()'s execution: The latter zeroes all
+ * @read_data_ok entries, which will make process_bulk_in() report false
+ * errors if executed. The mechanism relies on that xdev->error is assigned
+ * a non-zero value by report_io_error() prior to queueing wakeup_all(),
+ * which prevents bulk_in_work() from calling process_bulk_in().
+ *
+ * The fact that wakeup_all() and bulk_in_work() are queued on the same
+ * workqueue makes their concurrent execution very unlikely, however the
+ * kernel's API doesn't seem to ensure this strictly.
+ */
+
+static void wakeup_all(struct work_struct *work)
+{
+	int i;
+	struct xillyusb_dev *xdev = container_of(work, struct xillyusb_dev,
+						 wakeup_workitem);
+
+	mutex_lock(&xdev->process_in_mutex);
+
+	for (i = 0; i < xdev->num_channels; i++) {
+		struct xillyusb_channel *chan = &xdev->channels[i];
+
+		mutex_lock(&chan->lock);
+
+		if (chan->in_fifo) {
+			/*
+			 * Fake an EOF: Even if such arrives, it won't be
+			 * processed.
+			 */
+			chan->read_data_ok = 0;
+			wake_up_interruptible(&chan->in_fifo->waitq);
+		}
+
+		if (chan->out_ep)
+			wake_up_interruptible(&chan->out_ep->fifo.waitq);
+
+		mutex_unlock(&chan->lock);
+
+		wake_up_interruptible(&chan->flushq);
+	}
+
+	mutex_unlock(&xdev->process_in_mutex);
+
+	wake_up_interruptible(&xdev->msg_ep->fifo.waitq);
+
+	kref_put(&xdev->kref, cleanup_dev);
+}
+
+static void report_io_error(struct xillyusb_dev *xdev,
+			    int errcode)
+{
+	unsigned long flags;
+	bool do_once = false;
+
+	spin_lock_irqsave(&xdev->error_lock, flags);
+	if (!xdev->error) {
+		xdev->error = errcode;
+		do_once = true;
+	}
+	spin_unlock_irqrestore(&xdev->error_lock, flags);
+
+	if (do_once) {
+		kref_get(&xdev->kref); /* xdev is used by work item */
+		queue_work(xdev->workq, &xdev->wakeup_workitem);
+	}
+}
+
+/*
+ * safely_assign_in_fifo() changes the value of chan->in_fifo and ensures
+ * the previous pointer is never used after its return.
+ */
+
+static void safely_assign_in_fifo(struct xillyusb_channel *chan,
+				  struct xillyfifo *fifo)
+{
+	mutex_lock(&chan->lock);
+	chan->in_fifo = fifo;
+	mutex_unlock(&chan->lock);
+
+	flush_work(&chan->xdev->in_ep->workitem);
+}
+
+static void bulk_in_completer(struct urb *urb)
+{
+	struct xillybuffer *xb = urb->context;
+	struct xillyusb_endpoint *ep = xb->ep;
+	unsigned long flags;
+
+	if (urb->status) {
+		if (!(urb->status == -ENOENT ||
+		      urb->status == -ECONNRESET ||
+		      urb->status == -ESHUTDOWN))
+			report_io_error(ep->xdev, -EIO);
+
+		spin_lock_irqsave(&ep->buffers_lock, flags);
+		list_add_tail(&xb->entry, &ep->buffers);
+		ep->outstanding_urbs--;
+		spin_unlock_irqrestore(&ep->buffers_lock, flags);
+
+		return;
+	}
+
+	xb->len = urb->actual_length;
+
+	spin_lock_irqsave(&ep->buffers_lock, flags);
+	list_add_tail(&xb->entry, &ep->filled_buffers);
+	spin_unlock_irqrestore(&ep->buffers_lock, flags);
+
+	if (!ep->shutting_down)
+		queue_work(ep->xdev->workq, &ep->workitem);
+}
+
+static void bulk_out_completer(struct urb *urb)
+{
+	struct xillybuffer *xb = urb->context;
+	struct xillyusb_endpoint *ep = xb->ep;
+	unsigned long flags;
+
+	if (urb->status &&
+	    (!(urb->status == -ENOENT ||
+	       urb->status == -ECONNRESET ||
+	       urb->status == -ESHUTDOWN)))
+		report_io_error(ep->xdev, -EIO);
+
+	spin_lock_irqsave(&ep->buffers_lock, flags);
+	list_add_tail(&xb->entry, &ep->buffers);
+	ep->outstanding_urbs--;
+	spin_unlock_irqrestore(&ep->buffers_lock, flags);
+
+	if (!ep->shutting_down)
+		queue_work(ep->xdev->workq, &ep->workitem);
+}
+
+static void try_queue_bulk_in(struct xillyusb_endpoint *ep)
+{
+	struct xillyusb_dev *xdev = ep->xdev;
+	struct xillybuffer *xb;
+	struct urb *urb;
+
+	int rc;
+	unsigned long flags;
+	unsigned int bufsize = ep->buffer_size;
+
+	mutex_lock(&ep->ep_mutex);
+
+	if (ep->shutting_down || xdev->error)
+		goto done;
+
+	while (1) {
+		spin_lock_irqsave(&ep->buffers_lock, flags);
+
+		if (list_empty(&ep->buffers)) {
+			spin_unlock_irqrestore(&ep->buffers_lock, flags);
+			goto done;
+		}
+
+		xb = list_first_entry(&ep->buffers, struct xillybuffer, entry);
+		list_del(&xb->entry);
+		ep->outstanding_urbs++;
+
+		spin_unlock_irqrestore(&ep->buffers_lock, flags);
+
+		urb = usb_alloc_urb(0, GFP_KERNEL);
+		if (!urb) {
+			report_io_error(xdev, -ENOMEM);
+			goto relist;
+		}
+
+		usb_fill_bulk_urb(urb, xdev->udev,
+				  usb_rcvbulkpipe(xdev->udev, ep->ep_num),
+				  xb->buf, bufsize, bulk_in_completer, xb);
+
+		usb_anchor_urb(urb, &ep->anchor);
+
+		rc = usb_submit_urb(urb, GFP_KERNEL);
+
+		if (rc) {
+			report_io_error(xdev, (rc == -ENOMEM) ? -ENOMEM :
+					-EIO);
+			goto unanchor;
+		}
+
+		usb_free_urb(urb); /* This just decrements reference count */
+	}
+
+unanchor:
+	usb_unanchor_urb(urb);
+	usb_free_urb(urb);
+
+relist:
+	spin_lock_irqsave(&ep->buffers_lock, flags);
+	list_add_tail(&xb->entry, &ep->buffers);
+	ep->outstanding_urbs--;
+	spin_unlock_irqrestore(&ep->buffers_lock, flags);
+
+done:
+	mutex_unlock(&ep->ep_mutex);
+}
+
+static void try_queue_bulk_out(struct xillyusb_endpoint *ep)
+{
+	struct xillyfifo *fifo = &ep->fifo;
+	struct xillyusb_dev *xdev = ep->xdev;
+	struct xillybuffer *xb;
+	struct urb *urb;
+
+	int rc;
+	unsigned int fill;
+	unsigned long flags;
+	bool do_wake = false;
+
+	mutex_lock(&ep->ep_mutex);
+
+	if (ep->shutting_down || xdev->error)
+		goto done;
+
+	fill = READ_ONCE(fifo->fill) & ep->fill_mask;
+
+	while (1) {
+		int count;
+		unsigned int max_read;
+
+		spin_lock_irqsave(&ep->buffers_lock, flags);
+
+		/*
+		 * Race conditions might have the FIFO filled while the
+		 * endpoint is marked as drained here. That doesn't matter,
+		 * because the sole purpose of @drained is to ensure that
+		 * certain data has been sent on the USB channel before
+		 * shutting it down. Hence knowing that the FIFO appears
+		 * to be empty with no outstanding URBs at some moment
+		 * is good enough.
+		 */
+
+		if (!fill) {
+			ep->drained = !ep->outstanding_urbs;
+			if (ep->drained && ep->wake_on_drain)
+				do_wake = true;
+
+			spin_unlock_irqrestore(&ep->buffers_lock, flags);
+			goto done;
+		}
+
+		ep->drained = false;
+
+		if ((fill < ep->buffer_size && ep->outstanding_urbs) ||
+		    list_empty(&ep->buffers)) {
+			spin_unlock_irqrestore(&ep->buffers_lock, flags);
+			goto done;
+		}
+
+		xb = list_first_entry(&ep->buffers, struct xillybuffer, entry);
+		list_del(&xb->entry);
+		ep->outstanding_urbs++;
+
+		spin_unlock_irqrestore(&ep->buffers_lock, flags);
+
+		max_read = min(fill, ep->buffer_size);
+
+		count = fifo_read(&ep->fifo, xb->buf, max_read, xilly_memcpy);
+
+		/*
+		 * xilly_memcpy always returns 0 => fifo_read can't fail =>
+		 * count > 0
+		 */
+
+		urb = usb_alloc_urb(0, GFP_KERNEL);
+		if (!urb) {
+			report_io_error(xdev, -ENOMEM);
+			goto relist;
+		}
+
+		usb_fill_bulk_urb(urb, xdev->udev,
+				  usb_sndbulkpipe(xdev->udev, ep->ep_num),
+				  xb->buf, count, bulk_out_completer, xb);
+
+		usb_anchor_urb(urb, &ep->anchor);
+
+		rc = usb_submit_urb(urb, GFP_KERNEL);
+
+		if (rc) {
+			report_io_error(xdev, (rc == -ENOMEM) ? -ENOMEM :
+					-EIO);
+			goto unanchor;
+		}
+
+		usb_free_urb(urb); /* This just decrements reference count */
+
+		fill -= count;
+		do_wake = true;
+	}
+
+unanchor:
+	usb_unanchor_urb(urb);
+	usb_free_urb(urb);
+
+relist:
+	spin_lock_irqsave(&ep->buffers_lock, flags);
+	list_add_tail(&xb->entry, &ep->buffers);
+	ep->outstanding_urbs--;
+	spin_unlock_irqrestore(&ep->buffers_lock, flags);
+
+done:
+	mutex_unlock(&ep->ep_mutex);
+
+	if (do_wake)
+		wake_up_interruptible(&fifo->waitq);
+}
+
+static void bulk_out_work(struct work_struct *work)
+{
+	struct xillyusb_endpoint *ep = container_of(work,
+						    struct xillyusb_endpoint,
+						    workitem);
+	try_queue_bulk_out(ep);
+}
+
+static int process_in_opcode(struct xillyusb_dev *xdev,
+			     int opcode,
+			     int chan_num)
+{
+	struct xillyusb_channel *chan;
+	struct device *dev = xdev->dev;
+	int chan_idx = chan_num >> 1;
+
+	if (chan_idx >= xdev->num_channels) {
+		dev_err(dev, "Received illegal channel ID %d from FPGA\n",
+			chan_num);
+		return -EIO;
+	}
+
+	chan = &xdev->channels[chan_idx];
+
+	switch (opcode) {
+	case OPCODE_EOF:
+		if (!chan->read_data_ok) {
+			dev_err(dev, "Received unexpected EOF for channel %d\n",
+				chan_num);
+			return -EIO;
+		}
+
+		/*
+		 * A write memory barrier ensures that the FIFO's fill level
+		 * is visible before read_data_ok turns zero, so the data in
+		 * the FIFO isn't missed by the consumer.
+		 */
+		smp_wmb();
+		WRITE_ONCE(chan->read_data_ok, 0);
+		wake_up_interruptible(&chan->in_fifo->waitq);
+		break;
+
+	case OPCODE_REACHED_CHECKPOINT:
+		chan->flushing = 0;
+		wake_up_interruptible(&chan->flushq);
+		break;
+
+	case OPCODE_CANCELED_CHECKPOINT:
+		chan->canceled = 1;
+		wake_up_interruptible(&chan->flushq);
+		break;
+
+	default:
+		dev_err(dev, "Received illegal opcode %d from FPGA\n",
+			opcode);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static int process_bulk_in(struct xillybuffer *xb)
+{
+	struct xillyusb_endpoint *ep = xb->ep;
+	struct xillyusb_dev *xdev = ep->xdev;
+	struct device *dev = xdev->dev;
+	int dws = xb->len >> 2;
+	__le32 *p = xb->buf;
+	u32 ctrlword;
+	struct xillyusb_channel *chan;
+	struct xillyfifo *fifo;
+	int chan_num = 0, opcode;
+	int chan_idx;
+	int bytes, count, dwconsume;
+	int in_bytes_left = 0;
+	int rc;
+
+	if ((dws << 2) != xb->len) {
+		dev_err(dev, "Received BULK IN transfer with %d bytes, not a multiple of 4\n",
+			xb->len);
+		return -EIO;
+	}
+
+	if (xdev->in_bytes_left) {
+		bytes = min(xdev->in_bytes_left, dws << 2);
+		in_bytes_left = xdev->in_bytes_left - bytes;
+		chan_num = xdev->leftover_chan_num;
+		goto resume_leftovers;
+	}
+
+	while (dws) {
+		ctrlword = le32_to_cpu(*p++);
+		dws--;
+
+		chan_num = ctrlword & 0xfff;
+		count = (ctrlword >> 12) & 0x3ff;
+		opcode = (ctrlword >> 24) & 0xf;
+
+		if (opcode != OPCODE_DATA) {
+			unsigned int in_counter = xdev->in_counter++ & 0x3ff;
+
+			if (count != in_counter) {
+				dev_err(dev, "Expected opcode counter %d, got %d\n",
+					in_counter, count);
+				return -EIO;
+			}
+
+			rc = process_in_opcode(xdev, opcode, chan_num);
+
+			if (rc)
+				return rc;
+
+			continue;
+		}
+
+		bytes = min(count + 1, dws << 2);
+		in_bytes_left = count + 1 - bytes;
+
+resume_leftovers:
+		chan_idx = chan_num >> 1;
+
+		if (!(chan_num & 1) || chan_idx >= xdev->num_channels ||
+		    !xdev->channels[chan_idx].read_data_ok) {
+			dev_err(dev, "Received illegal channel ID %d from FPGA\n",
+				chan_num);
+			return -EIO;
+		}
+		chan = &xdev->channels[chan_idx];
+
+		fifo = chan->in_fifo;
+
+		if (unlikely(!fifo))
+			return -EIO; /* We got really unexpected data */
+
+		if (bytes != fifo_write(fifo, p, bytes, xilly_memcpy)) {
+			dev_err(dev, "Misbehaving FPGA overflowed an upstream FIFO!\n");
+			return -EIO;
+		}
+
+		wake_up_interruptible(&fifo->waitq);
+
+		dwconsume = (bytes + 3) >> 2;
+		dws -= dwconsume;
+		p += dwconsume;
+	}
+
+	xdev->in_bytes_left = in_bytes_left;
+	xdev->leftover_chan_num = chan_num;
+	return 0;
+}
+
+static void bulk_in_work(struct work_struct *work)
+{
+	struct xillyusb_endpoint *ep =
+		container_of(work, struct xillyusb_endpoint, workitem);
+	struct xillyusb_dev *xdev = ep->xdev;
+	unsigned long flags;
+	struct xillybuffer *xb;
+	bool consumed = false;
+	int rc = 0;
+
+	mutex_lock(&xdev->process_in_mutex);
+
+	spin_lock_irqsave(&ep->buffers_lock, flags);
+
+	while (1) {
+		if (rc || list_empty(&ep->filled_buffers)) {
+			spin_unlock_irqrestore(&ep->buffers_lock, flags);
+			mutex_unlock(&xdev->process_in_mutex);
+
+			if (rc)
+				report_io_error(xdev, rc);
+			else if (consumed)
+				try_queue_bulk_in(ep);
+
+			return;
+		}
+
+		xb = list_first_entry(&ep->filled_buffers, struct xillybuffer,
+				      entry);
+		list_del(&xb->entry);
+
+		spin_unlock_irqrestore(&ep->buffers_lock, flags);
+
+		consumed = true;
+
+		if (!xdev->error)
+			rc = process_bulk_in(xb);
+
+		spin_lock_irqsave(&ep->buffers_lock, flags);
+		list_add_tail(&xb->entry, &ep->buffers);
+		ep->outstanding_urbs--;
+	}
+}
+
+static int xillyusb_send_opcode(struct xillyusb_dev *xdev,
+				int chan_num, char opcode, u32 data)
+{
+	struct xillyusb_endpoint *ep = xdev->msg_ep;
+	struct xillyfifo *fifo = &ep->fifo;
+	__le32 msg[2];
+
+	int rc = 0;
+
+	msg[0] = cpu_to_le32((chan_num & 0xfff) |
+			     ((opcode & 0xf) << 24));
+	msg[1] = cpu_to_le32(data);
+
+	mutex_lock(&xdev->msg_mutex);
+
+	/*
+	 * The wait queue is woken with the interruptible variant, so the
+	 * wait function matches, however returning because of an interrupt
+	 * will mess things up considerably, in particular when the caller is
+	 * the release method. And the xdev->error part prevents being stuck
+	 * forever in the event of a bizarre hardware bug: Pull the USB plug.
+	 */
+
+	while (wait_event_interruptible(fifo->waitq,
+					fifo->fill <= (fifo->size - 8) ||
+					xdev->error))
+		; /* Empty loop */
+
+	if (xdev->error) {
+		rc = xdev->error;
+		goto unlock_done;
+	}
+
+	fifo_write(fifo, (void *)msg, 8, xilly_memcpy);
+
+	try_queue_bulk_out(ep);
+
+unlock_done:
+	mutex_unlock(&xdev->msg_mutex);
+
+	return rc;
+}
+
+/*
+ * Note that flush_downstream() merely waits for the data to arrive to
+ * the application logic at the FPGA -- unlike PCIe Xillybus' counterpart,
+ * it does nothing to make it happen (and neither is it necessary).
+ *
+ * This function is not reentrant for the same @chan, but this is covered
+ * by the fact that for any given @chan, it's called either by the open,
+ * write, llseek and flush fops methods, which can't run in parallel (and the
+ * write + flush and llseek method handlers are protected with out_mutex).
+ *
+ * chan->flushed is there to avoid multiple flushes at the same position,
+ * in particular as a result of programs that close the file descriptor
+ * e.g. after a dup2() for redirection.
+ */
+
+static int flush_downstream(struct xillyusb_channel *chan,
+			    long timeout,
+			    bool interruptible)
+{
+	struct xillyusb_dev *xdev = chan->xdev;
+	int chan_num = chan->chan_idx << 1;
+	long deadline, left_to_sleep;
+	int rc;
+
+	if (chan->flushed)
+		return 0;
+
+	deadline = jiffies + 1 + timeout;
+
+	if (chan->flushing) {
+		long cancel_deadline = jiffies + 1 + XILLY_RESPONSE_TIMEOUT;
+
+		chan->canceled = 0;
+		rc = xillyusb_send_opcode(xdev, chan_num,
+					  OPCODE_CANCEL_CHECKPOINT, 0);
+
+		if (rc)
+			return rc; /* Only real error, never -EINTR */
+
+		/* Ignoring interrupts. Cancellation must be handled */
+		while (!chan->canceled) {
+			left_to_sleep = cancel_deadline - ((long)jiffies);
+
+			if (left_to_sleep <= 0) {
+				report_io_error(xdev, -EIO);
+				return -EIO;
+			}
+
+			rc = wait_event_interruptible_timeout(chan->flushq,
+							      chan->canceled ||
+							      xdev->error,
+							      left_to_sleep);
+
+			if (xdev->error)
+				return xdev->error;
+		}
+	}
+
+	chan->flushing = 1;
+
+	/*
+	 * The checkpoint is given in terms of data elements, not bytes. As
+	 * a result, if less than an element's worth of data is stored in the
+	 * FIFO, it's not flushed, including the flush before closing, which
+	 * means that such data is lost. This is consistent with PCIe Xillybus.
+	 */
+
+	rc = xillyusb_send_opcode(xdev, chan_num,
+				  OPCODE_SET_CHECKPOINT,
+				  chan->out_bytes >>
+				  chan->out_log2_element_size);
+
+	if (rc)
+		return rc; /* Only real error, never -EINTR */
+
+	if (!timeout) {
+		while (chan->flushing) {
+			rc = wait_event_interruptible(chan->flushq,
+						      !chan->flushing ||
+						      xdev->error);
+			if (xdev->error)
+				return xdev->error;
+
+			if (interruptible && rc)
+				return -EINTR;
+		}
+
+		goto done;
+	}
+
+	while (chan->flushing) {
+		left_to_sleep = deadline - ((long)jiffies);
+
+		if (left_to_sleep <= 0)
+			return -ETIMEDOUT;
+
+		rc = wait_event_interruptible_timeout(chan->flushq,
+						      !chan->flushing ||
+						      xdev->error,
+						      left_to_sleep);
+
+		if (xdev->error)
+			return xdev->error;
+
+		if (interruptible && rc < 0)
+			return -EINTR;
+	}
+
+done:
+	chan->flushed = 1;
+	return 0;
+}
+
+/* request_read_anything(): Ask the FPGA for any little amount of data */
+static int request_read_anything(struct xillyusb_channel *chan,
+				 char opcode)
+{
+	struct xillyusb_dev *xdev = chan->xdev;
+	unsigned int sh = chan->in_log2_element_size;
+	int chan_num = (chan->chan_idx << 1) | 1;
+	u32 mercy = chan->in_consumed_bytes + (2 << sh) - 1;
+
+	return xillyusb_send_opcode(xdev, chan_num, opcode, mercy >> sh);
+}
+
+static int xillyusb_open(struct inode *inode, struct file *filp)
+{
+	struct xillyusb_dev *xdev;
+	struct xillyusb_channel *chan;
+	struct xillyfifo *in_fifo = NULL;
+	struct xillyusb_endpoint *out_ep = NULL;
+	int rc;
+	int index;
+
+	rc = xillybus_find_inode(inode, (void **)&xdev, &index);
+	if (rc)
+		return rc;
+
+	chan = &xdev->channels[index];
+	filp->private_data = chan;
+
+	mutex_lock(&chan->lock);
+
+	rc = -ENODEV;
+
+	if (xdev->error)
+		goto unmutex_fail;
+
+	if (((filp->f_mode & FMODE_READ) && !chan->readable) ||
+	    ((filp->f_mode & FMODE_WRITE) && !chan->writable))
+		goto unmutex_fail;
+
+	if ((filp->f_flags & O_NONBLOCK) && (filp->f_mode & FMODE_READ) &&
+	    chan->in_synchronous) {
+		dev_err(xdev->dev,
+			"open() failed: O_NONBLOCK not allowed for read on this device\n");
+		goto unmutex_fail;
+	}
+
+	if ((filp->f_flags & O_NONBLOCK) && (filp->f_mode & FMODE_WRITE) &&
+	    chan->out_synchronous) {
+		dev_err(xdev->dev,
+			"open() failed: O_NONBLOCK not allowed for write on this device\n");
+		goto unmutex_fail;
+	}
+
+	rc = -EBUSY;
+
+	if (((filp->f_mode & FMODE_READ) && chan->open_for_read) ||
+	    ((filp->f_mode & FMODE_WRITE) && chan->open_for_write))
+		goto unmutex_fail;
+
+	kref_get(&xdev->kref);
+
+	if (filp->f_mode & FMODE_READ)
+		chan->open_for_read = 1;
+
+	if (filp->f_mode & FMODE_WRITE)
+		chan->open_for_write = 1;
+
+	mutex_unlock(&chan->lock);
+
+	if (filp->f_mode & FMODE_WRITE) {
+		out_ep = endpoint_alloc(xdev,
+					(chan->chan_idx + 2) | USB_DIR_OUT,
+					bulk_out_work, BUF_SIZE_ORDER, BUFNUM);
+
+		if (!out_ep) {
+			rc = -ENOMEM;
+			goto unopen;
+		}
+
+		rc = fifo_init(&out_ep->fifo, chan->out_log2_fifo_size);
+
+		if (rc)
+			goto late_unopen;
+
+		out_ep->fill_mask = -(1 << chan->out_log2_element_size);
+		chan->out_bytes = 0;
+		chan->flushed = 0;
+
+		/*
+		 * Sending a flush request to a previously closed stream
+		 * effectively opens it, and also waits until the command is
+		 * confirmed by the FPGA. The latter is necessary because the
+		 * data is sent through a separate BULK OUT endpoint, and the
+		 * xHCI controller is free to reorder transmissions.
+		 *
+		 * This can't go wrong unless there's a serious hardware error
+		 * (or the computer is stuck for 500 ms?)
+		 */
+		rc = flush_downstream(chan, XILLY_RESPONSE_TIMEOUT, false);
+
+		if (rc == -ETIMEDOUT) {
+			rc = -EIO;
+			report_io_error(xdev, rc);
+		}
+
+		if (rc)
+			goto late_unopen;
+	}
+
+	if (filp->f_mode & FMODE_READ) {
+		in_fifo = kzalloc(sizeof(*in_fifo), GFP_KERNEL);
+
+		if (!in_fifo) {
+			rc = -ENOMEM;
+			goto late_unopen;
+		}
+
+		rc = fifo_init(in_fifo, chan->in_log2_fifo_size);
+
+		if (rc) {
+			kfree(in_fifo);
+			goto late_unopen;
+		}
+	}
+
+	mutex_lock(&chan->lock);
+	if (in_fifo) {
+		chan->in_fifo = in_fifo;
+		chan->read_data_ok = 1;
+	}
+	if (out_ep)
+		chan->out_ep = out_ep;
+	mutex_unlock(&chan->lock);
+
+	if (in_fifo) {
+		u32 in_checkpoint = 0;
+
+		if (!chan->in_synchronous)
+			in_checkpoint = in_fifo->size >>
+				chan->in_log2_element_size;
+
+		chan->in_consumed_bytes = 0;
+		chan->poll_used = 0;
+		chan->in_current_checkpoint = in_checkpoint;
+		rc = xillyusb_send_opcode(xdev, (chan->chan_idx << 1) | 1,
+					  OPCODE_SET_CHECKPOINT,
+					  in_checkpoint);
+
+		if (rc) /* Failure guarantees that opcode wasn't sent */
+			goto unfifo;
+
+		/*
+		 * In non-blocking mode, request the FPGA to send any data it
+		 * has right away. Otherwise, the first read() will always
+		 * return -EAGAIN, which is OK strictly speaking, but ugly.
+		 * Checking and unrolling if this fails isn't worth the
+		 * effort -- the error is propagated to the first read()
+		 * anyhow.
+		 */
+		if (filp->f_flags & O_NONBLOCK)
+			request_read_anything(chan, OPCODE_SET_PUSH);
+	}
+
+	return 0;
+
+unfifo:
+	chan->read_data_ok = 0;
+	safely_assign_in_fifo(chan, NULL);
+	fifo_mem_release(in_fifo);
+	kfree(in_fifo);
+
+	if (out_ep) {
+		mutex_lock(&chan->lock);
+		chan->out_ep = NULL;
+		mutex_unlock(&chan->lock);
+	}
+
+late_unopen:
+	if (out_ep)
+		endpoint_dealloc(out_ep);
+
+unopen:
+	mutex_lock(&chan->lock);
+
+	if (filp->f_mode & FMODE_READ)
+		chan->open_for_read = 0;
+
+	if (filp->f_mode & FMODE_WRITE)
+		chan->open_for_write = 0;
+
+	mutex_unlock(&chan->lock);
+
+	kref_put(&xdev->kref, cleanup_dev);
+
+	return rc;
+
+unmutex_fail:
+	mutex_unlock(&chan->lock);
+	return rc;
+}
+
+static ssize_t xillyusb_read(struct file *filp, char __user *userbuf,
+			     size_t count, loff_t *f_pos)
+{
+	struct xillyusb_channel *chan = filp->private_data;
+	struct xillyusb_dev *xdev = chan->xdev;
+	struct xillyfifo *fifo = chan->in_fifo;
+	int chan_num = (chan->chan_idx << 1) | 1;
+
+	long deadline, left_to_sleep;
+	int bytes_done = 0;
+	bool sent_set_push = false;
+	int rc;
+
+	deadline = jiffies + 1 + XILLY_RX_TIMEOUT;
+
+	rc = mutex_lock_interruptible(&chan->in_mutex);
+
+	if (rc)
+		return rc;
+
+	while (1) {
+		u32 fifo_checkpoint_bytes, complete_checkpoint_bytes;
+		u32 complete_checkpoint, fifo_checkpoint;
+		u32 checkpoint;
+		s32 diff, leap;
+		unsigned int sh = chan->in_log2_element_size;
+		bool checkpoint_for_complete;
+
+		rc = fifo_read(fifo, (__force void *)userbuf + bytes_done,
+			       count - bytes_done, xilly_copy_to_user);
+
+		if (rc < 0)
+			break;
+
+		bytes_done += rc;
+		chan->in_consumed_bytes += rc;
+
+		left_to_sleep = deadline - ((long)jiffies);
+
+		/*
+		 * Some 32-bit arithmetic that may wrap. Note that
+		 * complete_checkpoint is rounded up to the closest element
+		 * boundary, because the read() can't be completed otherwise.
+		 * fifo_checkpoint_bytes is rounded down, because it protects
+		 * in_fifo from overflowing.
+		 */
+
+		fifo_checkpoint_bytes = chan->in_consumed_bytes + fifo->size;
+		complete_checkpoint_bytes =
+			chan->in_consumed_bytes + count - bytes_done;
+
+		fifo_checkpoint = fifo_checkpoint_bytes >> sh;
+		complete_checkpoint =
+			(complete_checkpoint_bytes + (1 << sh) - 1) >> sh;
+
+		diff = (fifo_checkpoint - complete_checkpoint) << sh;
+
+		if (chan->in_synchronous && diff >= 0) {
+			checkpoint = complete_checkpoint;
+			checkpoint_for_complete = true;
+		} else {
+			checkpoint = fifo_checkpoint;
+			checkpoint_for_complete = false;
+		}
+
+		leap = (checkpoint - chan->in_current_checkpoint) << sh;
+
+		/*
+		 * To prevent flooding of OPCODE_SET_CHECKPOINT commands as
+		 * data is consumed, it's issued only if it moves the
+		 * checkpoint by at least an 8th of the FIFO's size, or if
+		 * it's necessary to complete the number of bytes requested by
+		 * the read() call.
+		 *
+		 * chan->read_data_ok is checked to spare an unnecessary
+		 * submission after receiving EOF, however it's harmless if
+		 * such slips away.
+		 */
+
+		if (chan->read_data_ok &&
+		    (leap > (fifo->size >> 3) ||
+		     (checkpoint_for_complete && leap > 0))) {
+			chan->in_current_checkpoint = checkpoint;
+			rc = xillyusb_send_opcode(xdev, chan_num,
+						  OPCODE_SET_CHECKPOINT,
+						  checkpoint);
+
+			if (rc)
+				break;
+		}
+
+		if (bytes_done == count ||
+		    (left_to_sleep <= 0 && bytes_done))
+			break;
+
+		/*
+		 * Reaching here means that the FIFO was empty when
+		 * fifo_read() returned, but not necessarily right now. Error
+		 * and EOF are checked and reported only now, so that no data
+		 * that managed its way to the FIFO is lost.
+		 */
+
+		if (!READ_ONCE(chan->read_data_ok)) { /* FPGA has sent EOF */
+			/* Has data slipped into the FIFO since fifo_read()? */
+			smp_rmb();
+			if (READ_ONCE(fifo->fill))
+				continue;
+
+			rc = 0;
+			break;
+		}
+
+		if (xdev->error) {
+			rc = xdev->error;
+			break;
+		}
+
+		if (filp->f_flags & O_NONBLOCK) {
+			rc = -EAGAIN;
+			break;
+		}
+
+		if (!sent_set_push) {
+			rc = xillyusb_send_opcode(xdev, chan_num,
+						  OPCODE_SET_PUSH,
+						  complete_checkpoint);
+
+			if (rc)
+				break;
+
+			sent_set_push = true;
+		}
+
+		if (left_to_sleep > 0) {
+			/*
+			 * Note that when xdev->error is set (e.g. when the
+			 * device is unplugged), read_data_ok turns zero and
+			 * fifo->waitq is awaken.
+			 * Therefore no special attention to xdev->error.
+			 */
+
+			rc = wait_event_interruptible_timeout
+				(fifo->waitq,
+				 fifo->fill || !chan->read_data_ok,
+				 left_to_sleep);
+		} else { /* bytes_done == 0 */
+			/* Tell FPGA to send anything it has */
+			rc = request_read_anything(chan, OPCODE_UPDATE_PUSH);
+
+			if (rc)
+				break;
+
+			rc = wait_event_interruptible
+				(fifo->waitq,
+				 fifo->fill || !chan->read_data_ok);
+		}
+
+		if (rc < 0) {
+			rc = -EINTR;
+			break;
+		}
+	}
+
+	if (((filp->f_flags & O_NONBLOCK) || chan->poll_used) &&
+	    !READ_ONCE(fifo->fill))
+		request_read_anything(chan, OPCODE_SET_PUSH);
+
+	mutex_unlock(&chan->in_mutex);
+
+	if (bytes_done)
+		return bytes_done;
+
+	return rc;
+}
+
+static int xillyusb_flush(struct file *filp, fl_owner_t id)
+{
+	struct xillyusb_channel *chan = filp->private_data;
+	int rc;
+
+	if (!(filp->f_mode & FMODE_WRITE))
+		return 0;
+
+	rc = mutex_lock_interruptible(&chan->out_mutex);
+
+	if (rc)
+		return rc;
+
+	/*
+	 * One second's timeout on flushing. Interrupts are ignored, because if
+	 * the user pressed CTRL-C, that interrupt will still be in flight by
+	 * the time we reach here, and the opportunity to flush is lost.
+	 */
+	rc = flush_downstream(chan, HZ, false);
+
+	mutex_unlock(&chan->out_mutex);
+
+	if (rc == -ETIMEDOUT) {
+		/* The things you do to use dev_warn() and not pr_warn() */
+		struct xillyusb_dev *xdev = chan->xdev;
+
+		mutex_lock(&chan->lock);
+		if (!xdev->error)
+			dev_warn(xdev->dev,
+				 "Timed out while flushing. Output data may be lost.\n");
+		mutex_unlock(&chan->lock);
+	}
+
+	return rc;
+}
+
+static ssize_t xillyusb_write(struct file *filp, const char __user *userbuf,
+			      size_t count, loff_t *f_pos)
+{
+	struct xillyusb_channel *chan = filp->private_data;
+	struct xillyusb_dev *xdev = chan->xdev;
+	struct xillyfifo *fifo = &chan->out_ep->fifo;
+	int rc;
+
+	rc = mutex_lock_interruptible(&chan->out_mutex);
+
+	if (rc)
+		return rc;
+
+	while (1) {
+		if (xdev->error) {
+			rc = xdev->error;
+			break;
+		}
+
+		if (count == 0)
+			break;
+
+		rc = fifo_write(fifo, (__force void *)userbuf, count,
+				xilly_copy_from_user);
+
+		if (rc != 0)
+			break;
+
+		if (filp->f_flags & O_NONBLOCK) {
+			rc = -EAGAIN;
+			break;
+		}
+
+		if (wait_event_interruptible
+		    (fifo->waitq,
+		     fifo->fill != fifo->size || xdev->error)) {
+			rc = -EINTR;
+			break;
+		}
+	}
+
+	if (rc < 0)
+		goto done;
+
+	chan->out_bytes += rc;
+
+	if (rc) {
+		try_queue_bulk_out(chan->out_ep);
+		chan->flushed = 0;
+	}
+
+	if (chan->out_synchronous) {
+		int flush_rc = flush_downstream(chan, 0, true);
+
+		if (flush_rc && !rc)
+			rc = flush_rc;
+	}
+
+done:
+	mutex_unlock(&chan->out_mutex);
+
+	return rc;
+}
+
+static int xillyusb_release(struct inode *inode, struct file *filp)
+{
+	struct xillyusb_channel *chan = filp->private_data;
+	struct xillyusb_dev *xdev = chan->xdev;
+	int rc_read = 0, rc_write = 0;
+
+	if (filp->f_mode & FMODE_READ) {
+		struct xillyfifo *in_fifo = chan->in_fifo;
+
+		rc_read = xillyusb_send_opcode(xdev, (chan->chan_idx << 1) | 1,
+					       OPCODE_CLOSE, 0);
+		/*
+		 * If rc_read is nonzero, xdev->error indicates a global
+		 * device error. The error is reported later, so that
+		 * resources are freed.
+		 *
+		 * Looping on wait_event_interruptible() kinda breaks the idea
+		 * of being interruptible, and this should have been
+		 * wait_event(). Only it's being waken with
+		 * wake_up_interruptible() for the sake of other uses. If
+		 * there's a global device error, chan->read_data_ok is
+		 * deasserted and the wait queue is awaken, so this is covered.
+		 */
+
+		while (wait_event_interruptible(in_fifo->waitq,
+						!chan->read_data_ok))
+			; /* Empty loop */
+
+		safely_assign_in_fifo(chan, NULL);
+		fifo_mem_release(in_fifo);
+		kfree(in_fifo);
+
+		mutex_lock(&chan->lock);
+		chan->open_for_read = 0;
+		mutex_unlock(&chan->lock);
+	}
+
+	if (filp->f_mode & FMODE_WRITE) {
+		struct xillyusb_endpoint *ep = chan->out_ep;
+		/*
+		 * chan->flushing isn't zeroed. If the pre-release flush timed
+		 * out, a cancel request will be sent before the next
+		 * OPCODE_SET_CHECKPOINT (i.e. when the file is opened again).
+		 * This is despite that the FPGA forgets about the checkpoint
+		 * request as the file closes. Still, in an exceptional race
+		 * condition, the FPGA could send an OPCODE_REACHED_CHECKPOINT
+		 * just before closing that would reach the host after the
+		 * file has re-opened.
+		 */
+
+		mutex_lock(&chan->lock);
+		chan->out_ep = NULL;
+		mutex_unlock(&chan->lock);
+
+		endpoint_quiesce(ep);
+		endpoint_dealloc(ep);
+
+		/* See comments on rc_read above */
+		rc_write = xillyusb_send_opcode(xdev, chan->chan_idx << 1,
+						OPCODE_CLOSE, 0);
+
+		mutex_lock(&chan->lock);
+		chan->open_for_write = 0;
+		mutex_unlock(&chan->lock);
+	}
+
+	kref_put(&xdev->kref, cleanup_dev);
+
+	return rc_read ? rc_read : rc_write;
+}
+
+/*
+ * Xillybus' API allows device nodes to be seekable, giving the user
+ * application access to a RAM array on the FPGA (or logic emulating it).
+ */
+
+static loff_t xillyusb_llseek(struct file *filp, loff_t offset, int whence)
+{
+	struct xillyusb_channel *chan = filp->private_data;
+	struct xillyusb_dev *xdev = chan->xdev;
+	loff_t pos = filp->f_pos;
+	int rc = 0;
+	unsigned int log2_element_size = chan->readable ?
+		chan->in_log2_element_size : chan->out_log2_element_size;
+
+	/*
+	 * Take both mutexes not allowing interrupts, since it seems like
+	 * common applications don't expect an -EINTR here. Besides, multiple
+	 * access to a single file descriptor on seekable devices is a mess
+	 * anyhow.
+	 */
+
+	mutex_lock(&chan->out_mutex);
+	mutex_lock(&chan->in_mutex);
+
+	switch (whence) {
+	case SEEK_SET:
+		pos = offset;
+		break;
+	case SEEK_CUR:
+		pos += offset;
+		break;
+	case SEEK_END:
+		pos = offset; /* Going to the end => to the beginning */
+		break;
+	default:
+		rc = -EINVAL;
+		goto end;
+	}
+
+	/* In any case, we must finish on an element boundary */
+	if (pos & ((1 << log2_element_size) - 1)) {
+		rc = -EINVAL;
+		goto end;
+	}
+
+	rc = xillyusb_send_opcode(xdev, chan->chan_idx << 1,
+				  OPCODE_SET_ADDR,
+				  pos >> log2_element_size);
+
+	if (rc)
+		goto end;
+
+	if (chan->writable) {
+		chan->flushed = 0;
+		rc = flush_downstream(chan, HZ, false);
+	}
+
+end:
+	mutex_unlock(&chan->out_mutex);
+	mutex_unlock(&chan->in_mutex);
+
+	if (rc) /* Return error after releasing mutexes */
+		return rc;
+
+	filp->f_pos = pos;
+
+	return pos;
+}
+
+static __poll_t xillyusb_poll(struct file *filp, poll_table *wait)
+{
+	struct xillyusb_channel *chan = filp->private_data;
+	__poll_t mask = 0;
+
+	if (chan->in_fifo)
+		poll_wait(filp, &chan->in_fifo->waitq, wait);
+
+	if (chan->out_ep)
+		poll_wait(filp, &chan->out_ep->fifo.waitq, wait);
+
+	/*
+	 * If this is the first time poll() is called, and the file is
+	 * readable, set the relevant flag. Also tell the FPGA to send all it
+	 * has, to kickstart the mechanism that ensures there's always some
+	 * data in in_fifo unless the stream is dry end-to-end. Note that the
+	 * first poll() may not return a EPOLLIN, even if there's data on the
+	 * FPGA. Rather, the data will arrive soon, and trigger the relevant
+	 * wait queue.
+	 */
+
+	if (!chan->poll_used && chan->in_fifo) {
+		chan->poll_used = 1;
+		request_read_anything(chan, OPCODE_SET_PUSH);
+	}
+
+	/*
+	 * poll() won't play ball regarding read() channels which
+	 * are synchronous. Allowing that will create situations where data has
+	 * been delivered at the FPGA, and users expecting select() to wake up,
+	 * which it may not. So make it never work.
+	 */
+
+	if (chan->in_fifo && !chan->in_synchronous &&
+	    (READ_ONCE(chan->in_fifo->fill) || !chan->read_data_ok))
+		mask |= EPOLLIN | EPOLLRDNORM;
+
+	if (chan->out_ep &&
+	    (READ_ONCE(chan->out_ep->fifo.fill) != chan->out_ep->fifo.size))
+		mask |= EPOLLOUT | EPOLLWRNORM;
+
+	if (chan->xdev->error)
+		mask |= EPOLLERR;
+
+	return mask;
+}
+
+static const struct file_operations xillyusb_fops = {
+	.owner      = THIS_MODULE,
+	.read       = xillyusb_read,
+	.write      = xillyusb_write,
+	.open       = xillyusb_open,
+	.flush      = xillyusb_flush,
+	.release    = xillyusb_release,
+	.llseek     = xillyusb_llseek,
+	.poll       = xillyusb_poll,
+};
+
+static int xillyusb_setup_base_eps(struct xillyusb_dev *xdev)
+{
+	xdev->msg_ep = endpoint_alloc(xdev, MSG_EP_NUM | USB_DIR_OUT,
+				      bulk_out_work, 1, 2);
+	if (!xdev->msg_ep)
+		return -ENOMEM;
+
+	if (fifo_init(&xdev->msg_ep->fifo, 13)) /* 8 kiB */
+		goto dealloc;
+
+	xdev->msg_ep->fill_mask = -8; /* 8 bytes granularity */
+
+	xdev->in_ep = endpoint_alloc(xdev, IN_EP_NUM | USB_DIR_IN,
+				     bulk_in_work, BUF_SIZE_ORDER, BUFNUM);
+	if (!xdev->in_ep)
+		goto dealloc;
+
+	try_queue_bulk_in(xdev->in_ep);
+
+	return 0;
+
+dealloc:
+	endpoint_dealloc(xdev->msg_ep); /* Also frees FIFO mem if allocated */
+	xdev->msg_ep = NULL;
+	return -ENOMEM;
+}
+
+static int setup_channels(struct xillyusb_dev *xdev,
+			  __le16 *chandesc,
+			  int num_channels)
+{
+	struct xillyusb_channel *chan;
+	int i;
+
+	chan = kcalloc(num_channels, sizeof(*chan), GFP_KERNEL);
+	if (!chan)
+		return -ENOMEM;
+
+	xdev->channels = chan;
+
+	for (i = 0; i < num_channels; i++, chan++) {
+		unsigned int in_desc = le16_to_cpu(*chandesc++);
+		unsigned int out_desc = le16_to_cpu(*chandesc++);
+
+		chan->xdev = xdev;
+		mutex_init(&chan->in_mutex);
+		mutex_init(&chan->out_mutex);
+		mutex_init(&chan->lock);
+		init_waitqueue_head(&chan->flushq);
+
+		chan->chan_idx = i;
+
+		if (in_desc & 0x80) { /* Entry is valid */
+			chan->readable = 1;
+			chan->in_synchronous = !!(in_desc & 0x40);
+			chan->in_seekable = !!(in_desc & 0x20);
+			chan->in_log2_element_size = in_desc & 0x0f;
+			chan->in_log2_fifo_size = ((in_desc >> 8) & 0x1f) + 16;
+		}
+
+		/*
+		 * A downstream channel should never exist above index 13,
+		 * as it would request a nonexistent BULK endpoint > 15.
+		 * In the peculiar case that it does, it's ignored silently.
+		 */
+
+		if ((out_desc & 0x80) && i < 14) { /* Entry is valid */
+			chan->writable = 1;
+			chan->out_synchronous = !!(out_desc & 0x40);
+			chan->out_seekable = !!(out_desc & 0x20);
+			chan->out_log2_element_size = out_desc & 0x0f;
+			chan->out_log2_fifo_size =
+				((out_desc >> 8) & 0x1f) + 16;
+		}
+	}
+
+	return 0;
+}
+
+static int xillyusb_discovery(struct usb_interface *interface)
+{
+	int rc;
+	struct xillyusb_dev *xdev = usb_get_intfdata(interface);
+	__le16 bogus_chandesc[2];
+	struct xillyfifo idt_fifo;
+	struct xillyusb_channel *chan;
+	unsigned int idt_len, names_offset;
+	unsigned char *idt;
+	int num_channels;
+
+	rc = xillyusb_send_opcode(xdev, ~0, OPCODE_QUIESCE, 0);
+
+	if (rc) {
+		dev_err(&interface->dev, "Failed to send quiesce request. Aborting.\n");
+		return rc;
+	}
+
+	/* Phase I: Set up one fake upstream channel and obtain IDT */
+
+	/* Set up a fake IDT with one async IN stream */
+	bogus_chandesc[0] = cpu_to_le16(0x80);
+	bogus_chandesc[1] = cpu_to_le16(0);
+
+	rc = setup_channels(xdev, bogus_chandesc, 1);
+
+	if (rc)
+		return rc;
+
+	rc = fifo_init(&idt_fifo, LOG2_IDT_FIFO_SIZE);
+
+	if (rc)
+		return rc;
+
+	chan = xdev->channels;
+
+	chan->in_fifo = &idt_fifo;
+	chan->read_data_ok = 1;
+
+	xdev->num_channels = 1;
+
+	rc = xillyusb_send_opcode(xdev, ~0, OPCODE_REQ_IDT, 0);
+
+	if (rc) {
+		dev_err(&interface->dev, "Failed to send IDT request. Aborting.\n");
+		goto unfifo;
+	}
+
+	rc = wait_event_interruptible_timeout(idt_fifo.waitq,
+					      !chan->read_data_ok,
+					      XILLY_RESPONSE_TIMEOUT);
+
+	if (xdev->error) {
+		rc = xdev->error;
+		goto unfifo;
+	}
+
+	if (rc < 0) {
+		rc = -EINTR; /* Interrupt on probe method? Interesting. */
+		goto unfifo;
+	}
+
+	if (chan->read_data_ok) {
+		rc = -ETIMEDOUT;
+		dev_err(&interface->dev, "No response from FPGA. Aborting.\n");
+		goto unfifo;
+	}
+
+	idt_len = READ_ONCE(idt_fifo.fill);
+	idt = kmalloc(idt_len, GFP_KERNEL);
+
+	if (!idt) {
+		rc = -ENOMEM;
+		goto unfifo;
+	}
+
+	fifo_read(&idt_fifo, idt, idt_len, xilly_memcpy);
+
+	if (crc32_le(~0, idt, idt_len) != 0) {
+		dev_err(&interface->dev, "IDT failed CRC check. Aborting.\n");
+		rc = -ENODEV;
+		goto unidt;
+	}
+
+	if (*idt > 0x90) {
+		dev_err(&interface->dev, "No support for IDT version 0x%02x. Maybe the xillyusb driver needs an upgrade. Aborting.\n",
+			(int)*idt);
+		rc = -ENODEV;
+		goto unidt;
+	}
+
+	/* Phase II: Set up the streams as defined in IDT */
+
+	num_channels = le16_to_cpu(*((__le16 *)(idt + 1)));
+	names_offset = 3 + num_channels * 4;
+	idt_len -= 4; /* Exclude CRC */
+
+	if (idt_len < names_offset) {
+		dev_err(&interface->dev, "IDT too short. This is exceptionally weird, because its CRC is OK\n");
+		rc = -ENODEV;
+		goto unidt;
+	}
+
+	rc = setup_channels(xdev, (void *)idt + 3, num_channels);
+
+	if (rc)
+		goto unidt;
+
+	/*
+	 * Except for wildly misbehaving hardware, or if it was disconnected
+	 * just after responding with the IDT, there is no reason for any
+	 * work item to be running now. To be sure that xdev->channels
+	 * is updated on anything that might run in parallel, flush the
+	 * workqueue, which rarely does anything.
+	 */
+	flush_workqueue(xdev->workq);
+
+	xdev->num_channels = num_channels;
+
+	fifo_mem_release(&idt_fifo);
+	kfree(chan);
+
+	rc = xillybus_init_chrdev(&interface->dev, &xillyusb_fops,
+				  THIS_MODULE, xdev,
+				  idt + names_offset,
+				  idt_len - names_offset,
+				  num_channels,
+				  xillyname, true);
+
+	kfree(idt);
+
+	return rc;
+
+unidt:
+	kfree(idt);
+
+unfifo:
+	safely_assign_in_fifo(chan, NULL);
+	fifo_mem_release(&idt_fifo);
+
+	return rc;
+}
+
+static int xillyusb_probe(struct usb_interface *interface,
+			  const struct usb_device_id *id)
+{
+	struct xillyusb_dev *xdev;
+	int rc;
+
+	xdev = kzalloc(sizeof(*xdev), GFP_KERNEL);
+	if (!xdev)
+		return -ENOMEM;
+
+	kref_init(&xdev->kref);
+	mutex_init(&xdev->process_in_mutex);
+	mutex_init(&xdev->msg_mutex);
+
+	xdev->udev = usb_get_dev(interface_to_usbdev(interface));
+	xdev->dev = &interface->dev;
+	xdev->error = 0;
+	spin_lock_init(&xdev->error_lock);
+	xdev->in_counter = 0;
+	xdev->in_bytes_left = 0;
+	xdev->workq = alloc_workqueue(xillyname, WQ_HIGHPRI, 0);
+
+	if (!xdev->workq) {
+		dev_err(&interface->dev, "Failed to allocate work queue\n");
+		rc = -ENOMEM;
+		goto fail;
+	}
+
+	INIT_WORK(&xdev->wakeup_workitem, wakeup_all);
+
+	usb_set_intfdata(interface, xdev);
+
+	rc = xillyusb_setup_base_eps(xdev);
+	if (rc)
+		goto fail;
+
+	rc = xillyusb_discovery(interface);
+	if (rc)
+		goto latefail;
+
+	return 0;
+
+latefail:
+	endpoint_quiesce(xdev->in_ep);
+	endpoint_quiesce(xdev->msg_ep);
+
+fail:
+	usb_set_intfdata(interface, NULL);
+	kref_put(&xdev->kref, cleanup_dev);
+	return rc;
+}
+
+static void xillyusb_disconnect(struct usb_interface *interface)
+{
+	struct xillyusb_dev *xdev = usb_get_intfdata(interface);
+	struct xillyusb_endpoint *msg_ep = xdev->msg_ep;
+	struct xillyfifo *fifo = &msg_ep->fifo;
+	int rc;
+	int i;
+
+	xillybus_cleanup_chrdev(xdev, &interface->dev);
+
+	/*
+	 * Try to send OPCODE_QUIESCE, which will fail silently if the device
+	 * was disconnected, but makes sense on module unload.
+	 */
+
+	msg_ep->wake_on_drain = true;
+	xillyusb_send_opcode(xdev, ~0, OPCODE_QUIESCE, 0);
+
+	/*
+	 * If the device has been disconnected, sending the opcode causes
+	 * a global device error with xdev->error, if such error didn't
+	 * occur earlier. Hence timing out means that the USB link is fine,
+	 * but somehow the message wasn't sent. Should never happen.
+	 */
+
+	rc = wait_event_interruptible_timeout(fifo->waitq,
+					      msg_ep->drained || xdev->error,
+					      XILLY_RESPONSE_TIMEOUT);
+
+	if (!rc)
+		dev_err(&interface->dev,
+			"Weird timeout condition on sending quiesce request.\n");
+
+	report_io_error(xdev, -ENODEV); /* Discourage further activity */
+
+	/*
+	 * This device driver is declared with soft_unbind set, or else
+	 * sending OPCODE_QUIESCE above would always fail. The price is
+	 * that the USB framework didn't kill outstanding URBs, so it has
+	 * to be done explicitly before returning from this call.
+	 */
+
+	for (i = 0; i < xdev->num_channels; i++) {
+		struct xillyusb_channel *chan = &xdev->channels[i];
+
+		/*
+		 * Lock taken to prevent chan->out_ep from changing. It also
+		 * ensures xillyusb_open() and xillyusb_flush() don't access
+		 * xdev->dev after being nullified below.
+		 */
+		mutex_lock(&chan->lock);
+		if (chan->out_ep)
+			endpoint_quiesce(chan->out_ep);
+		mutex_unlock(&chan->lock);
+	}
+
+	endpoint_quiesce(xdev->in_ep);
+	endpoint_quiesce(xdev->msg_ep);
+
+	usb_set_intfdata(interface, NULL);
+
+	xdev->dev = NULL;
+
+	kref_put(&xdev->kref, cleanup_dev);
+}
+
+static struct usb_driver xillyusb_driver = {
+	.name = xillyname,
+	.id_table = xillyusb_table,
+	.probe = xillyusb_probe,
+	.disconnect = xillyusb_disconnect,
+	.soft_unbind = 1,
+};
+
+static int __init xillyusb_init(void)
+{
+	int rc = 0;
+
+	if (LOG2_INITIAL_FIFO_BUF_SIZE > PAGE_SHIFT)
+		fifo_buf_order = LOG2_INITIAL_FIFO_BUF_SIZE - PAGE_SHIFT;
+	else
+		fifo_buf_order = 0;
+
+	rc = usb_register(&xillyusb_driver);
+
+	return rc;
+}
+
+static void __exit xillyusb_exit(void)
+{
+	usb_deregister(&xillyusb_driver);
+}
+
+module_init(xillyusb_init);
+module_exit(xillyusb_exit);