diff options
Diffstat (limited to 'drivers/char/xillybus')
| -rw-r--r-- | drivers/char/xillybus/Kconfig | 52 | ||||
| -rw-r--r-- | drivers/char/xillybus/Makefile | 10 | ||||
| -rw-r--r-- | drivers/char/xillybus/xillybus.h | 126 | ||||
| -rw-r--r-- | drivers/char/xillybus/xillybus_class.c | 256 | ||||
| -rw-r--r-- | drivers/char/xillybus/xillybus_class.h | 30 | ||||
| -rw-r--r-- | drivers/char/xillybus/xillybus_core.c | 1991 | ||||
| -rw-r--r-- | drivers/char/xillybus/xillybus_of.c | 86 | ||||
| -rw-r--r-- | drivers/char/xillybus/xillybus_pcie.c | 127 | ||||
| -rw-r--r-- | drivers/char/xillybus/xillyusb.c | 2277 |
9 files changed, 4955 insertions, 0 deletions
diff --git a/drivers/char/xillybus/Kconfig b/drivers/char/xillybus/Kconfig new file mode 100644 index 0000000000..f51d533390 --- /dev/null +++ b/drivers/char/xillybus/Kconfig @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Xillybus devices +# + +config XILLYBUS_CLASS + tristate + +config XILLYBUS + tristate "Xillybus generic FPGA interface" + depends on PCI || OF + select CRC32 + select XILLYBUS_CLASS + help + Xillybus is a generic interface for peripherals designed on + programmable logic (FPGA). The driver probes the hardware for + its capabilities, and creates device files accordingly. + + If unsure, say N. + +if XILLYBUS + +config XILLYBUS_PCIE + tristate "Xillybus over PCIe" + depends on PCI_MSI + help + Set to M if you want Xillybus to use PCI Express for communicating + with the FPGA. The module will be called xillybus_pcie. + +config XILLYBUS_OF + tristate "Xillybus over Device Tree" + depends on OF && HAS_DMA && HAS_IOMEM + help + Set to M if you want Xillybus to find its resources from the + Open Firmware Flattened Device Tree. If the target is an embedded + system, say M. The module will be called xillybus_of. + +endif # if XILLYBUS + +# XILLYUSB doesn't depend on XILLYBUS + +config XILLYUSB + tristate "XillyUSB: Xillybus generic FPGA interface for USB" + depends on USB + select CRC32 + select XILLYBUS_CLASS + help + XillyUSB is the Xillybus variant which uses USB for communicating + with the FPGA. + + Set to M if you want Xillybus to use USB for communicating with + the FPGA. The module will be called xillyusb. diff --git a/drivers/char/xillybus/Makefile b/drivers/char/xillybus/Makefile new file mode 100644 index 0000000000..16f31d0320 --- /dev/null +++ b/drivers/char/xillybus/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for Xillybus driver +# + +obj-$(CONFIG_XILLYBUS_CLASS) += xillybus_class.o +obj-$(CONFIG_XILLYBUS) += xillybus_core.o +obj-$(CONFIG_XILLYBUS_PCIE) += xillybus_pcie.o +obj-$(CONFIG_XILLYBUS_OF) += xillybus_of.o +obj-$(CONFIG_XILLYUSB) += xillyusb.o diff --git a/drivers/char/xillybus/xillybus.h b/drivers/char/xillybus/xillybus.h new file mode 100644 index 0000000000..51de7cbc57 --- /dev/null +++ b/drivers/char/xillybus/xillybus.h @@ -0,0 +1,126 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * linux/drivers/misc/xillybus.h + * + * Copyright 2011 Xillybus Ltd, http://xillybus.com + * + * Header file for the Xillybus FPGA/host framework. + */ + +#ifndef __XILLYBUS_H +#define __XILLYBUS_H + +#include <linux/list.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/interrupt.h> +#include <linux/sched.h> +#include <linux/cdev.h> +#include <linux/spinlock.h> +#include <linux/mutex.h> +#include <linux/workqueue.h> + +struct xilly_endpoint_hardware; + +struct xilly_buffer { + void *addr; + dma_addr_t dma_addr; + int end_offset; /* Counting elements, not bytes */ +}; + +struct xilly_idt_handle { + unsigned char *chandesc; + unsigned char *names; + int names_len; + int entries; +}; + +/* + * Read-write confusion: wr_* and rd_* notation sticks to FPGA view, so + * wr_* buffers are those consumed by read(), since the FPGA writes to them + * and vice versa. + */ + +struct xilly_channel { + struct xilly_endpoint *endpoint; + int chan_num; + int log2_element_size; + int seekable; + + struct xilly_buffer **wr_buffers; /* FPGA writes, driver reads! */ + int num_wr_buffers; + unsigned int wr_buf_size; /* In bytes */ + int wr_fpga_buf_idx; + int wr_host_buf_idx; + int wr_host_buf_pos; + int wr_empty; + int wr_ready; /* Significant only when wr_empty == 1 */ + int wr_sleepy; + int wr_eof; + int wr_hangup; + spinlock_t wr_spinlock; + struct mutex wr_mutex; + wait_queue_head_t wr_wait; + wait_queue_head_t wr_ready_wait; + int wr_ref_count; + int wr_synchronous; + int wr_allow_partial; + int wr_exclusive_open; + int wr_supports_nonempty; + + struct xilly_buffer **rd_buffers; /* FPGA reads, driver writes! */ + int num_rd_buffers; + unsigned int rd_buf_size; /* In bytes */ + int rd_fpga_buf_idx; + int rd_host_buf_pos; + int rd_host_buf_idx; + int rd_full; + spinlock_t rd_spinlock; + struct mutex rd_mutex; + wait_queue_head_t rd_wait; + int rd_ref_count; + int rd_allow_partial; + int rd_synchronous; + int rd_exclusive_open; + struct delayed_work rd_workitem; + unsigned char rd_leftovers[4]; +}; + +struct xilly_endpoint { + struct device *dev; + struct module *owner; + + int dma_using_dac; /* =1 if 64-bit DMA is used, =0 otherwise. */ + __iomem void *registers; + int fatal_error; + + struct mutex register_mutex; + wait_queue_head_t ep_wait; + + int num_channels; /* EXCLUDING message buffer */ + struct xilly_channel **channels; + int msg_counter; + int failed_messages; + int idtlen; + + u32 *msgbuf_addr; + dma_addr_t msgbuf_dma_addr; + unsigned int msg_buf_size; +}; + +struct xilly_mapping { + struct device *device; + dma_addr_t dma_addr; + size_t size; + int direction; +}; + +irqreturn_t xillybus_isr(int irq, void *data); + +struct xilly_endpoint *xillybus_init_endpoint(struct device *dev); + +int xillybus_endpoint_discovery(struct xilly_endpoint *endpoint); + +void xillybus_endpoint_remove(struct xilly_endpoint *endpoint); + +#endif /* __XILLYBUS_H */ diff --git a/drivers/char/xillybus/xillybus_class.c b/drivers/char/xillybus/xillybus_class.c new file mode 100644 index 0000000000..c92a628e38 --- /dev/null +++ b/drivers/char/xillybus/xillybus_class.c @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2021 Xillybus Ltd, http://xillybus.com + * + * Driver for the Xillybus class + */ + +#include <linux/types.h> +#include <linux/module.h> +#include <linux/device.h> +#include <linux/fs.h> +#include <linux/cdev.h> +#include <linux/slab.h> +#include <linux/list.h> +#include <linux/mutex.h> + +#include "xillybus_class.h" + +MODULE_DESCRIPTION("Driver for Xillybus class"); +MODULE_AUTHOR("Eli Billauer, Xillybus Ltd."); +MODULE_ALIAS("xillybus_class"); +MODULE_LICENSE("GPL v2"); + +static DEFINE_MUTEX(unit_mutex); +static LIST_HEAD(unit_list); +static const struct class xillybus_class = { + .name = "xillybus", +}; + +#define UNITNAMELEN 16 + +struct xilly_unit { + struct list_head list_entry; + void *private_data; + + struct cdev *cdev; + char name[UNITNAMELEN]; + int major; + int lowest_minor; + int num_nodes; +}; + +int xillybus_init_chrdev(struct device *dev, + const struct file_operations *fops, + struct module *owner, + void *private_data, + unsigned char *idt, unsigned int len, + int num_nodes, + const char *prefix, bool enumerate) +{ + int rc; + dev_t mdev; + int i; + char devname[48]; + + struct device *device; + size_t namelen; + struct xilly_unit *unit, *u; + + unit = kzalloc(sizeof(*unit), GFP_KERNEL); + + if (!unit) + return -ENOMEM; + + mutex_lock(&unit_mutex); + + if (!enumerate) + snprintf(unit->name, UNITNAMELEN, "%s", prefix); + + for (i = 0; enumerate; i++) { + snprintf(unit->name, UNITNAMELEN, "%s_%02d", + prefix, i); + + enumerate = false; + list_for_each_entry(u, &unit_list, list_entry) + if (!strcmp(unit->name, u->name)) { + enumerate = true; + break; + } + } + + rc = alloc_chrdev_region(&mdev, 0, num_nodes, unit->name); + + if (rc) { + dev_warn(dev, "Failed to obtain major/minors"); + goto fail_obtain; + } + + unit->major = MAJOR(mdev); + unit->lowest_minor = MINOR(mdev); + unit->num_nodes = num_nodes; + unit->private_data = private_data; + + unit->cdev = cdev_alloc(); + if (!unit->cdev) { + rc = -ENOMEM; + goto unregister_chrdev; + } + unit->cdev->ops = fops; + unit->cdev->owner = owner; + + rc = cdev_add(unit->cdev, MKDEV(unit->major, unit->lowest_minor), + unit->num_nodes); + if (rc) { + dev_err(dev, "Failed to add cdev.\n"); + /* kobject_put() is normally done by cdev_del() */ + kobject_put(&unit->cdev->kobj); + goto unregister_chrdev; + } + + for (i = 0; i < num_nodes; i++) { + namelen = strnlen(idt, len); + + if (namelen == len) { + dev_err(dev, "IDT's list of names is too short. This is exceptionally weird, because its CRC is OK\n"); + rc = -ENODEV; + goto unroll_device_create; + } + + snprintf(devname, sizeof(devname), "%s_%s", + unit->name, idt); + + len -= namelen + 1; + idt += namelen + 1; + + device = device_create(&xillybus_class, + NULL, + MKDEV(unit->major, + i + unit->lowest_minor), + NULL, + "%s", devname); + + if (IS_ERR(device)) { + dev_err(dev, "Failed to create %s device. Aborting.\n", + devname); + rc = -ENODEV; + goto unroll_device_create; + } + } + + if (len) { + dev_err(dev, "IDT's list of names is too long. This is exceptionally weird, because its CRC is OK\n"); + rc = -ENODEV; + goto unroll_device_create; + } + + list_add_tail(&unit->list_entry, &unit_list); + + dev_info(dev, "Created %d device files.\n", num_nodes); + + mutex_unlock(&unit_mutex); + + return 0; + +unroll_device_create: + for (i--; i >= 0; i--) + device_destroy(&xillybus_class, MKDEV(unit->major, + i + unit->lowest_minor)); + + cdev_del(unit->cdev); + +unregister_chrdev: + unregister_chrdev_region(MKDEV(unit->major, unit->lowest_minor), + unit->num_nodes); + +fail_obtain: + mutex_unlock(&unit_mutex); + + kfree(unit); + + return rc; +} +EXPORT_SYMBOL(xillybus_init_chrdev); + +void xillybus_cleanup_chrdev(void *private_data, + struct device *dev) +{ + int minor; + struct xilly_unit *unit = NULL, *iter; + + mutex_lock(&unit_mutex); + + list_for_each_entry(iter, &unit_list, list_entry) + if (iter->private_data == private_data) { + unit = iter; + break; + } + + if (!unit) { + dev_err(dev, "Weird bug: Failed to find unit\n"); + mutex_unlock(&unit_mutex); + return; + } + + for (minor = unit->lowest_minor; + minor < (unit->lowest_minor + unit->num_nodes); + minor++) + device_destroy(&xillybus_class, MKDEV(unit->major, minor)); + + cdev_del(unit->cdev); + + unregister_chrdev_region(MKDEV(unit->major, unit->lowest_minor), + unit->num_nodes); + + dev_info(dev, "Removed %d device files.\n", + unit->num_nodes); + + list_del(&unit->list_entry); + kfree(unit); + + mutex_unlock(&unit_mutex); +} +EXPORT_SYMBOL(xillybus_cleanup_chrdev); + +int xillybus_find_inode(struct inode *inode, + void **private_data, int *index) +{ + int minor = iminor(inode); + int major = imajor(inode); + struct xilly_unit *unit = NULL, *iter; + + mutex_lock(&unit_mutex); + + list_for_each_entry(iter, &unit_list, list_entry) + if (iter->major == major && + minor >= iter->lowest_minor && + minor < (iter->lowest_minor + iter->num_nodes)) { + unit = iter; + break; + } + + if (!unit) { + mutex_unlock(&unit_mutex); + return -ENODEV; + } + + *private_data = unit->private_data; + *index = minor - unit->lowest_minor; + + mutex_unlock(&unit_mutex); + return 0; +} +EXPORT_SYMBOL(xillybus_find_inode); + +static int __init xillybus_class_init(void) +{ + return class_register(&xillybus_class); +} + +static void __exit xillybus_class_exit(void) +{ + class_unregister(&xillybus_class); +} + +module_init(xillybus_class_init); +module_exit(xillybus_class_exit); diff --git a/drivers/char/xillybus/xillybus_class.h b/drivers/char/xillybus/xillybus_class.h new file mode 100644 index 0000000000..5dbfdfc95c --- /dev/null +++ b/drivers/char/xillybus/xillybus_class.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2021 Xillybus Ltd, http://www.xillybus.com + * + * Header file for the Xillybus class + */ + +#ifndef __XILLYBUS_CLASS_H +#define __XILLYBUS_CLASS_H + +#include <linux/types.h> +#include <linux/device.h> +#include <linux/fs.h> +#include <linux/module.h> + +int xillybus_init_chrdev(struct device *dev, + const struct file_operations *fops, + struct module *owner, + void *private_data, + unsigned char *idt, unsigned int len, + int num_nodes, + const char *prefix, bool enumerate); + +void xillybus_cleanup_chrdev(void *private_data, + struct device *dev); + +int xillybus_find_inode(struct inode *inode, + void **private_data, int *index); + +#endif /* __XILLYBUS_CLASS_H */ diff --git a/drivers/char/xillybus/xillybus_core.c b/drivers/char/xillybus/xillybus_core.c new file mode 100644 index 0000000000..11b7c47492 --- /dev/null +++ b/drivers/char/xillybus/xillybus_core.c @@ -0,0 +1,1991 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/drivers/misc/xillybus_core.c + * + * Copyright 2011 Xillybus Ltd, http://xillybus.com + * + * Driver for the Xillybus FPGA/host framework. + * + * This driver interfaces with a special IP core in an FPGA, setting up + * a pipe between a hardware FIFO in the programmable logic and a device + * file in the host. The number of such pipes and their attributes are + * set up on the logic. This driver detects these automatically and + * creates the device files accordingly. + */ + +#include <linux/list.h> +#include <linux/device.h> +#include <linux/module.h> +#include <linux/io.h> +#include <linux/dma-mapping.h> +#include <linux/interrupt.h> +#include <linux/sched.h> +#include <linux/fs.h> +#include <linux/spinlock.h> +#include <linux/mutex.h> +#include <linux/crc32.h> +#include <linux/poll.h> +#include <linux/delay.h> +#include <linux/slab.h> +#include <linux/workqueue.h> +#include "xillybus.h" +#include "xillybus_class.h" + +MODULE_DESCRIPTION("Xillybus core functions"); +MODULE_AUTHOR("Eli Billauer, Xillybus Ltd."); +MODULE_ALIAS("xillybus_core"); +MODULE_LICENSE("GPL v2"); + +/* General timeout is 100 ms, rx timeout is 10 ms */ +#define XILLY_RX_TIMEOUT (10*HZ/1000) +#define XILLY_TIMEOUT (100*HZ/1000) + +#define fpga_msg_ctrl_reg 0x0008 +#define fpga_dma_control_reg 0x0020 +#define fpga_dma_bufno_reg 0x0024 +#define fpga_dma_bufaddr_lowaddr_reg 0x0028 +#define fpga_dma_bufaddr_highaddr_reg 0x002c +#define fpga_buf_ctrl_reg 0x0030 +#define fpga_buf_offset_reg 0x0034 +#define fpga_endian_reg 0x0040 + +#define XILLYMSG_OPCODE_RELEASEBUF 1 +#define XILLYMSG_OPCODE_QUIESCEACK 2 +#define XILLYMSG_OPCODE_FIFOEOF 3 +#define XILLYMSG_OPCODE_FATAL_ERROR 4 +#define XILLYMSG_OPCODE_NONEMPTY 5 + +static const char xillyname[] = "xillybus"; + +static struct workqueue_struct *xillybus_wq; + +/* + * Locking scheme: Mutexes protect invocations of character device methods. + * If both locks are taken, wr_mutex is taken first, rd_mutex second. + * + * wr_spinlock protects wr_*_buf_idx, wr_empty, wr_sleepy, wr_ready and the + * buffers' end_offset fields against changes made by IRQ handler (and in + * theory, other file request handlers, but the mutex handles that). Nothing + * else. + * They are held for short direct memory manipulations. Needless to say, + * no mutex locking is allowed when a spinlock is held. + * + * rd_spinlock does the same with rd_*_buf_idx, rd_empty and end_offset. + * + * register_mutex is endpoint-specific, and is held when non-atomic + * register operations are performed. wr_mutex and rd_mutex may be + * held when register_mutex is taken, but none of the spinlocks. Note that + * register_mutex doesn't protect against sporadic buf_ctrl_reg writes + * which are unrelated to buf_offset_reg, since they are harmless. + * + * Blocking on the wait queues is allowed with mutexes held, but not with + * spinlocks. + * + * Only interruptible blocking is allowed on mutexes and wait queues. + * + * All in all, the locking order goes (with skips allowed, of course): + * wr_mutex -> rd_mutex -> register_mutex -> wr_spinlock -> rd_spinlock + */ + +static void malformed_message(struct xilly_endpoint *endpoint, u32 *buf) +{ + int opcode; + int msg_channel, msg_bufno, msg_data, msg_dir; + + opcode = (buf[0] >> 24) & 0xff; + msg_dir = buf[0] & 1; + msg_channel = (buf[0] >> 1) & 0x7ff; + msg_bufno = (buf[0] >> 12) & 0x3ff; + msg_data = buf[1] & 0xfffffff; + + dev_warn(endpoint->dev, + "Malformed message (skipping): opcode=%d, channel=%03x, dir=%d, bufno=%03x, data=%07x\n", + opcode, msg_channel, msg_dir, msg_bufno, msg_data); +} + +/* + * xillybus_isr assumes the interrupt is allocated exclusively to it, + * which is the natural case MSI and several other hardware-oriented + * interrupts. Sharing is not allowed. + */ + +irqreturn_t xillybus_isr(int irq, void *data) +{ + struct xilly_endpoint *ep = data; + u32 *buf; + unsigned int buf_size; + int i; + int opcode; + unsigned int msg_channel, msg_bufno, msg_data, msg_dir; + struct xilly_channel *channel; + + buf = ep->msgbuf_addr; + buf_size = ep->msg_buf_size/sizeof(u32); + + dma_sync_single_for_cpu(ep->dev, ep->msgbuf_dma_addr, + ep->msg_buf_size, DMA_FROM_DEVICE); + + for (i = 0; i < buf_size; i += 2) { + if (((buf[i+1] >> 28) & 0xf) != ep->msg_counter) { + malformed_message(ep, &buf[i]); + dev_warn(ep->dev, + "Sending a NACK on counter %x (instead of %x) on entry %d\n", + ((buf[i+1] >> 28) & 0xf), + ep->msg_counter, + i/2); + + if (++ep->failed_messages > 10) { + dev_err(ep->dev, + "Lost sync with interrupt messages. Stopping.\n"); + } else { + dma_sync_single_for_device(ep->dev, + ep->msgbuf_dma_addr, + ep->msg_buf_size, + DMA_FROM_DEVICE); + + iowrite32(0x01, /* Message NACK */ + ep->registers + fpga_msg_ctrl_reg); + } + return IRQ_HANDLED; + } else if (buf[i] & (1 << 22)) /* Last message */ + break; + } + + if (i >= buf_size) { + dev_err(ep->dev, "Bad interrupt message. Stopping.\n"); + return IRQ_HANDLED; + } + + buf_size = i + 2; + + for (i = 0; i < buf_size; i += 2) { /* Scan through messages */ + opcode = (buf[i] >> 24) & 0xff; + + msg_dir = buf[i] & 1; + msg_channel = (buf[i] >> 1) & 0x7ff; + msg_bufno = (buf[i] >> 12) & 0x3ff; + msg_data = buf[i+1] & 0xfffffff; + + switch (opcode) { + case XILLYMSG_OPCODE_RELEASEBUF: + if ((msg_channel > ep->num_channels) || + (msg_channel == 0)) { + malformed_message(ep, &buf[i]); + break; + } + + channel = ep->channels[msg_channel]; + + if (msg_dir) { /* Write channel */ + if (msg_bufno >= channel->num_wr_buffers) { + malformed_message(ep, &buf[i]); + break; + } + spin_lock(&channel->wr_spinlock); + channel->wr_buffers[msg_bufno]->end_offset = + msg_data; + channel->wr_fpga_buf_idx = msg_bufno; + channel->wr_empty = 0; + channel->wr_sleepy = 0; + spin_unlock(&channel->wr_spinlock); + + wake_up_interruptible(&channel->wr_wait); + + } else { + /* Read channel */ + + if (msg_bufno >= channel->num_rd_buffers) { + malformed_message(ep, &buf[i]); + break; + } + + spin_lock(&channel->rd_spinlock); + channel->rd_fpga_buf_idx = msg_bufno; + channel->rd_full = 0; + spin_unlock(&channel->rd_spinlock); + + wake_up_interruptible(&channel->rd_wait); + if (!channel->rd_synchronous) + queue_delayed_work( + xillybus_wq, + &channel->rd_workitem, + XILLY_RX_TIMEOUT); + } + + break; + case XILLYMSG_OPCODE_NONEMPTY: + if ((msg_channel > ep->num_channels) || + (msg_channel == 0) || (!msg_dir) || + !ep->channels[msg_channel]->wr_supports_nonempty) { + malformed_message(ep, &buf[i]); + break; + } + + channel = ep->channels[msg_channel]; + + if (msg_bufno >= channel->num_wr_buffers) { + malformed_message(ep, &buf[i]); + break; + } + spin_lock(&channel->wr_spinlock); + if (msg_bufno == channel->wr_host_buf_idx) + channel->wr_ready = 1; + spin_unlock(&channel->wr_spinlock); + + wake_up_interruptible(&channel->wr_ready_wait); + + break; + case XILLYMSG_OPCODE_QUIESCEACK: + ep->idtlen = msg_data; + wake_up_interruptible(&ep->ep_wait); + + break; + case XILLYMSG_OPCODE_FIFOEOF: + if ((msg_channel > ep->num_channels) || + (msg_channel == 0) || (!msg_dir) || + !ep->channels[msg_channel]->num_wr_buffers) { + malformed_message(ep, &buf[i]); + break; + } + channel = ep->channels[msg_channel]; + spin_lock(&channel->wr_spinlock); + channel->wr_eof = msg_bufno; + channel->wr_sleepy = 0; + + channel->wr_hangup = channel->wr_empty && + (channel->wr_host_buf_idx == msg_bufno); + + spin_unlock(&channel->wr_spinlock); + + wake_up_interruptible(&channel->wr_wait); + + break; + case XILLYMSG_OPCODE_FATAL_ERROR: + ep->fatal_error = 1; + wake_up_interruptible(&ep->ep_wait); /* For select() */ + dev_err(ep->dev, + "FPGA reported a fatal error. This means that the low-level communication with the device has failed. This hardware problem is most likely unrelated to Xillybus (neither kernel module nor FPGA core), but reports are still welcome. All I/O is aborted.\n"); + break; + default: + malformed_message(ep, &buf[i]); + break; + } + } + + dma_sync_single_for_device(ep->dev, ep->msgbuf_dma_addr, + ep->msg_buf_size, DMA_FROM_DEVICE); + + ep->msg_counter = (ep->msg_counter + 1) & 0xf; + ep->failed_messages = 0; + iowrite32(0x03, ep->registers + fpga_msg_ctrl_reg); /* Message ACK */ + + return IRQ_HANDLED; +} +EXPORT_SYMBOL(xillybus_isr); + +/* + * A few trivial memory management functions. + * NOTE: These functions are used only on probe and remove, and therefore + * no locks are applied! + */ + +static void xillybus_autoflush(struct work_struct *work); + +struct xilly_alloc_state { + void *salami; + int left_of_salami; + int nbuffer; + enum dma_data_direction direction; + u32 regdirection; +}; + +static void xilly_unmap(void *ptr) +{ + struct xilly_mapping *data = ptr; + + dma_unmap_single(data->device, data->dma_addr, + data->size, data->direction); + + kfree(ptr); +} + +static int xilly_map_single(struct xilly_endpoint *ep, + void *ptr, + size_t size, + int direction, + dma_addr_t *ret_dma_handle + ) +{ + dma_addr_t addr; + struct xilly_mapping *this; + + this = kzalloc(sizeof(*this), GFP_KERNEL); + if (!this) + return -ENOMEM; + + addr = dma_map_single(ep->dev, ptr, size, direction); + + if (dma_mapping_error(ep->dev, addr)) { + kfree(this); + return -ENODEV; + } + + this->device = ep->dev; + this->dma_addr = addr; + this->size = size; + this->direction = direction; + + *ret_dma_handle = addr; + + return devm_add_action_or_reset(ep->dev, xilly_unmap, this); +} + +static int xilly_get_dma_buffers(struct xilly_endpoint *ep, + struct xilly_alloc_state *s, + struct xilly_buffer **buffers, + int bufnum, int bytebufsize) +{ + int i, rc; + dma_addr_t dma_addr; + struct device *dev = ep->dev; + struct xilly_buffer *this_buffer = NULL; /* Init to silence warning */ + + if (buffers) { /* Not the message buffer */ + this_buffer = devm_kcalloc(dev, bufnum, + sizeof(struct xilly_buffer), + GFP_KERNEL); + if (!this_buffer) + return -ENOMEM; + } + + for (i = 0; i < bufnum; i++) { + /* + * Buffers are expected in descending size order, so there + * is either enough space for this buffer or none at all. + */ + + if ((s->left_of_salami < bytebufsize) && + (s->left_of_salami > 0)) { + dev_err(ep->dev, + "Corrupt buffer allocation in IDT. Aborting.\n"); + return -ENODEV; + } + + if (s->left_of_salami == 0) { + int allocorder, allocsize; + + allocsize = PAGE_SIZE; + allocorder = 0; + while (bytebufsize > allocsize) { + allocsize *= 2; + allocorder++; + } + + s->salami = (void *) devm_get_free_pages( + dev, + GFP_KERNEL | __GFP_DMA32 | __GFP_ZERO, + allocorder); + if (!s->salami) + return -ENOMEM; + + s->left_of_salami = allocsize; + } + + rc = xilly_map_single(ep, s->salami, + bytebufsize, s->direction, + &dma_addr); + if (rc) + return rc; + + iowrite32((u32) (dma_addr & 0xffffffff), + ep->registers + fpga_dma_bufaddr_lowaddr_reg); + iowrite32(((u32) ((((u64) dma_addr) >> 32) & 0xffffffff)), + ep->registers + fpga_dma_bufaddr_highaddr_reg); + + if (buffers) { /* Not the message buffer */ + this_buffer->addr = s->salami; + this_buffer->dma_addr = dma_addr; + buffers[i] = this_buffer++; + + iowrite32(s->regdirection | s->nbuffer++, + ep->registers + fpga_dma_bufno_reg); + } else { + ep->msgbuf_addr = s->salami; + ep->msgbuf_dma_addr = dma_addr; + ep->msg_buf_size = bytebufsize; + + iowrite32(s->regdirection, + ep->registers + fpga_dma_bufno_reg); + } + + s->left_of_salami -= bytebufsize; + s->salami += bytebufsize; + } + return 0; +} + +static int xilly_setupchannels(struct xilly_endpoint *ep, + unsigned char *chandesc, + int entries) +{ + struct device *dev = ep->dev; + int i, entry, rc; + struct xilly_channel *channel; + int channelnum, bufnum, bufsize, format, is_writebuf; + int bytebufsize; + int synchronous, allowpartial, exclusive_open, seekable; + int supports_nonempty; + int msg_buf_done = 0; + + struct xilly_alloc_state rd_alloc = { + .salami = NULL, + .left_of_salami = 0, + .nbuffer = 1, + .direction = DMA_TO_DEVICE, + .regdirection = 0, + }; + + struct xilly_alloc_state wr_alloc = { + .salami = NULL, + .left_of_salami = 0, + .nbuffer = 1, + .direction = DMA_FROM_DEVICE, + .regdirection = 0x80000000, + }; + + channel = devm_kcalloc(dev, ep->num_channels, + sizeof(struct xilly_channel), GFP_KERNEL); + if (!channel) + return -ENOMEM; + + ep->channels = devm_kcalloc(dev, ep->num_channels + 1, + sizeof(struct xilly_channel *), + GFP_KERNEL); + if (!ep->channels) + return -ENOMEM; + + ep->channels[0] = NULL; /* Channel 0 is message buf. */ + + /* Initialize all channels with defaults */ + + for (i = 1; i <= ep->num_channels; i++) { + channel->wr_buffers = NULL; + channel->rd_buffers = NULL; + channel->num_wr_buffers = 0; + channel->num_rd_buffers = 0; + channel->wr_fpga_buf_idx = -1; + channel->wr_host_buf_idx = 0; + channel->wr_host_buf_pos = 0; + channel->wr_empty = 1; + channel->wr_ready = 0; + channel->wr_sleepy = 1; + channel->rd_fpga_buf_idx = 0; + channel->rd_host_buf_idx = 0; + channel->rd_host_buf_pos = 0; + channel->rd_full = 0; + channel->wr_ref_count = 0; + channel->rd_ref_count = 0; + + spin_lock_init(&channel->wr_spinlock); + spin_lock_init(&channel->rd_spinlock); + mutex_init(&channel->wr_mutex); + mutex_init(&channel->rd_mutex); + init_waitqueue_head(&channel->rd_wait); + init_waitqueue_head(&channel->wr_wait); + init_waitqueue_head(&channel->wr_ready_wait); + + INIT_DELAYED_WORK(&channel->rd_workitem, xillybus_autoflush); + + channel->endpoint = ep; + channel->chan_num = i; + + channel->log2_element_size = 0; + + ep->channels[i] = channel++; + } + + for (entry = 0; entry < entries; entry++, chandesc += 4) { + struct xilly_buffer **buffers = NULL; + + is_writebuf = chandesc[0] & 0x01; + channelnum = (chandesc[0] >> 1) | ((chandesc[1] & 0x0f) << 7); + format = (chandesc[1] >> 4) & 0x03; + allowpartial = (chandesc[1] >> 6) & 0x01; + synchronous = (chandesc[1] >> 7) & 0x01; + bufsize = 1 << (chandesc[2] & 0x1f); + bufnum = 1 << (chandesc[3] & 0x0f); + exclusive_open = (chandesc[2] >> 7) & 0x01; + seekable = (chandesc[2] >> 6) & 0x01; + supports_nonempty = (chandesc[2] >> 5) & 0x01; + + if ((channelnum > ep->num_channels) || + ((channelnum == 0) && !is_writebuf)) { + dev_err(ep->dev, + "IDT requests channel out of range. Aborting.\n"); + return -ENODEV; + } + + channel = ep->channels[channelnum]; /* NULL for msg channel */ + + if (!is_writebuf || channelnum > 0) { + channel->log2_element_size = ((format > 2) ? + 2 : format); + + bytebufsize = bufsize * + (1 << channel->log2_element_size); + + buffers = devm_kcalloc(dev, bufnum, + sizeof(struct xilly_buffer *), + GFP_KERNEL); + if (!buffers) + return -ENOMEM; + } else { + bytebufsize = bufsize << 2; + } + + if (!is_writebuf) { + channel->num_rd_buffers = bufnum; + channel->rd_buf_size = bytebufsize; + channel->rd_allow_partial = allowpartial; + channel->rd_synchronous = synchronous; + channel->rd_exclusive_open = exclusive_open; + channel->seekable = seekable; + + channel->rd_buffers = buffers; + rc = xilly_get_dma_buffers(ep, &rd_alloc, buffers, + bufnum, bytebufsize); + } else if (channelnum > 0) { + channel->num_wr_buffers = bufnum; + channel->wr_buf_size = bytebufsize; + + channel->seekable = seekable; + channel->wr_supports_nonempty = supports_nonempty; + + channel->wr_allow_partial = allowpartial; + channel->wr_synchronous = synchronous; + channel->wr_exclusive_open = exclusive_open; + + channel->wr_buffers = buffers; + rc = xilly_get_dma_buffers(ep, &wr_alloc, buffers, + bufnum, bytebufsize); + } else { + rc = xilly_get_dma_buffers(ep, &wr_alloc, NULL, + bufnum, bytebufsize); + msg_buf_done++; + } + + if (rc) + return -ENOMEM; + } + + if (!msg_buf_done) { + dev_err(ep->dev, + "Corrupt IDT: No message buffer. Aborting.\n"); + return -ENODEV; + } + return 0; +} + +static int xilly_scan_idt(struct xilly_endpoint *endpoint, + struct xilly_idt_handle *idt_handle) +{ + int count = 0; + unsigned char *idt = endpoint->channels[1]->wr_buffers[0]->addr; + unsigned char *end_of_idt = idt + endpoint->idtlen - 4; + unsigned char *scan; + int len; + + scan = idt + 1; + idt_handle->names = scan; + + while ((scan <= end_of_idt) && *scan) { + while ((scan <= end_of_idt) && *scan++) + /* Do nothing, just scan thru string */; + count++; + } + + idt_handle->names_len = scan - idt_handle->names; + + scan++; + + if (scan > end_of_idt) { + dev_err(endpoint->dev, + "IDT device name list overflow. Aborting.\n"); + return -ENODEV; + } + idt_handle->chandesc = scan; + + len = endpoint->idtlen - (3 + ((int) (scan - idt))); + + if (len & 0x03) { + dev_err(endpoint->dev, + "Corrupt IDT device name list. Aborting.\n"); + return -ENODEV; + } + + idt_handle->entries = len >> 2; + endpoint->num_channels = count; + + return 0; +} + +static int xilly_obtain_idt(struct xilly_endpoint *endpoint) +{ + struct xilly_channel *channel; + unsigned char *version; + long t; + + channel = endpoint->channels[1]; /* This should be generated ad-hoc */ + + channel->wr_sleepy = 1; + + iowrite32(1 | + (3 << 24), /* Opcode 3 for channel 0 = Send IDT */ + endpoint->registers + fpga_buf_ctrl_reg); + + t = wait_event_interruptible_timeout(channel->wr_wait, + (!channel->wr_sleepy), + XILLY_TIMEOUT); + + if (t <= 0) { + dev_err(endpoint->dev, "Failed to obtain IDT. Aborting.\n"); + + if (endpoint->fatal_error) + return -EIO; + + return -ENODEV; + } + + dma_sync_single_for_cpu(channel->endpoint->dev, + channel->wr_buffers[0]->dma_addr, + channel->wr_buf_size, + DMA_FROM_DEVICE); + + if (channel->wr_buffers[0]->end_offset != endpoint->idtlen) { + dev_err(endpoint->dev, + "IDT length mismatch (%d != %d). Aborting.\n", + channel->wr_buffers[0]->end_offset, endpoint->idtlen); + return -ENODEV; + } + + if (crc32_le(~0, channel->wr_buffers[0]->addr, + endpoint->idtlen+1) != 0) { + dev_err(endpoint->dev, "IDT failed CRC check. Aborting.\n"); + return -ENODEV; + } + + version = channel->wr_buffers[0]->addr; + + /* Check version number. Reject anything above 0x82. */ + if (*version > 0x82) { + dev_err(endpoint->dev, + "No support for IDT version 0x%02x. Maybe the xillybus driver needs an upgrade. Aborting.\n", + *version); + return -ENODEV; + } + + return 0; +} + +static ssize_t xillybus_read(struct file *filp, char __user *userbuf, + size_t count, loff_t *f_pos) +{ + ssize_t rc; + unsigned long flags; + int bytes_done = 0; + int no_time_left = 0; + long deadline, left_to_sleep; + struct xilly_channel *channel = filp->private_data; + + int empty, reached_eof, exhausted, ready; + /* Initializations are there only to silence warnings */ + + int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0; + int waiting_bufidx; + + if (channel->endpoint->fatal_error) + return -EIO; + + deadline = jiffies + 1 + XILLY_RX_TIMEOUT; + + rc = mutex_lock_interruptible(&channel->wr_mutex); + if (rc) + return rc; + + while (1) { /* Note that we may drop mutex within this loop */ + int bytes_to_do = count - bytes_done; + + spin_lock_irqsave(&channel->wr_spinlock, flags); + + empty = channel->wr_empty; + ready = !empty || channel->wr_ready; + + if (!empty) { + bufidx = channel->wr_host_buf_idx; + bufpos = channel->wr_host_buf_pos; + howmany = ((channel->wr_buffers[bufidx]->end_offset + + 1) << channel->log2_element_size) + - bufpos; + + /* Update wr_host_* to its post-operation state */ + if (howmany > bytes_to_do) { + bufferdone = 0; + + howmany = bytes_to_do; + channel->wr_host_buf_pos += howmany; + } else { + bufferdone = 1; + + channel->wr_host_buf_pos = 0; + + if (bufidx == channel->wr_fpga_buf_idx) { + channel->wr_empty = 1; + channel->wr_sleepy = 1; + channel->wr_ready = 0; + } + + if (bufidx >= (channel->num_wr_buffers - 1)) + channel->wr_host_buf_idx = 0; + else + channel->wr_host_buf_idx++; + } + } + + /* + * Marking our situation after the possible changes above, + * for use after releasing the spinlock. + * + * empty = empty before change + * exhasted = empty after possible change + */ + + reached_eof = channel->wr_empty && + (channel->wr_host_buf_idx == channel->wr_eof); + channel->wr_hangup = reached_eof; + exhausted = channel->wr_empty; + waiting_bufidx = channel->wr_host_buf_idx; + + spin_unlock_irqrestore(&channel->wr_spinlock, flags); + + if (!empty) { /* Go on, now without the spinlock */ + + if (bufpos == 0) /* Position zero means it's virgin */ + dma_sync_single_for_cpu(channel->endpoint->dev, + channel->wr_buffers[bufidx]->dma_addr, + channel->wr_buf_size, + DMA_FROM_DEVICE); + + if (copy_to_user( + userbuf, + channel->wr_buffers[bufidx]->addr + + bufpos, howmany)) + rc = -EFAULT; + + userbuf += howmany; + bytes_done += howmany; + + if (bufferdone) { + dma_sync_single_for_device(channel->endpoint->dev, + channel->wr_buffers[bufidx]->dma_addr, + channel->wr_buf_size, + DMA_FROM_DEVICE); + + /* + * Tell FPGA the buffer is done with. It's an + * atomic operation to the FPGA, so what + * happens with other channels doesn't matter, + * and the certain channel is protected with + * the channel-specific mutex. + */ + + iowrite32(1 | (channel->chan_num << 1) | + (bufidx << 12), + channel->endpoint->registers + + fpga_buf_ctrl_reg); + } + + if (rc) { + mutex_unlock(&channel->wr_mutex); + return rc; + } + } + + /* This includes a zero-count return = EOF */ + if ((bytes_done >= count) || reached_eof) + break; + + if (!exhausted) + continue; /* More in RAM buffer(s)? Just go on. */ + + if ((bytes_done > 0) && + (no_time_left || + (channel->wr_synchronous && channel->wr_allow_partial))) + break; + + /* + * Nonblocking read: The "ready" flag tells us that the FPGA + * has data to send. In non-blocking mode, if it isn't on, + * just return. But if there is, we jump directly to the point + * where we ask for the FPGA to send all it has, and wait + * until that data arrives. So in a sense, we *do* block in + * nonblocking mode, but only for a very short time. + */ + + if (!no_time_left && (filp->f_flags & O_NONBLOCK)) { + if (bytes_done > 0) + break; + + if (ready) + goto desperate; + + rc = -EAGAIN; + break; + } + + if (!no_time_left || (bytes_done > 0)) { + /* + * Note that in case of an element-misaligned read + * request, offsetlimit will include the last element, + * which will be partially read from. + */ + int offsetlimit = ((count - bytes_done) - 1) >> + channel->log2_element_size; + int buf_elements = channel->wr_buf_size >> + channel->log2_element_size; + + /* + * In synchronous mode, always send an offset limit. + * Just don't send a value too big. + */ + + if (channel->wr_synchronous) { + /* Don't request more than one buffer */ + if (channel->wr_allow_partial && + (offsetlimit >= buf_elements)) + offsetlimit = buf_elements - 1; + + /* Don't request more than all buffers */ + if (!channel->wr_allow_partial && + (offsetlimit >= + (buf_elements * channel->num_wr_buffers))) + offsetlimit = buf_elements * + channel->num_wr_buffers - 1; + } + + /* + * In asynchronous mode, force early flush of a buffer + * only if that will allow returning a full count. The + * "offsetlimit < ( ... )" rather than "<=" excludes + * requesting a full buffer, which would obviously + * cause a buffer transmission anyhow + */ + + if (channel->wr_synchronous || + (offsetlimit < (buf_elements - 1))) { + mutex_lock(&channel->endpoint->register_mutex); + + iowrite32(offsetlimit, + channel->endpoint->registers + + fpga_buf_offset_reg); + + iowrite32(1 | (channel->chan_num << 1) | + (2 << 24) | /* 2 = offset limit */ + (waiting_bufidx << 12), + channel->endpoint->registers + + fpga_buf_ctrl_reg); + + mutex_unlock(&channel->endpoint-> + register_mutex); + } + } + + /* + * If partial completion is disallowed, there is no point in + * timeout sleeping. Neither if no_time_left is set and + * there's no data. + */ + + if (!channel->wr_allow_partial || + (no_time_left && (bytes_done == 0))) { + /* + * This do-loop will run more than once if another + * thread reasserted wr_sleepy before we got the mutex + * back, so we try again. + */ + + do { + mutex_unlock(&channel->wr_mutex); + + if (wait_event_interruptible( + channel->wr_wait, + (!channel->wr_sleepy))) + goto interrupted; + + if (mutex_lock_interruptible( + &channel->wr_mutex)) + goto interrupted; + } while (channel->wr_sleepy); + + continue; + +interrupted: /* Mutex is not held if got here */ + if (channel->endpoint->fatal_error) + return -EIO; + if (bytes_done) + return bytes_done; + if (filp->f_flags & O_NONBLOCK) + return -EAGAIN; /* Don't admit snoozing */ + return -EINTR; + } + + left_to_sleep = deadline - ((long) jiffies); + + /* + * If our time is out, skip the waiting. We may miss wr_sleepy + * being deasserted but hey, almost missing the train is like + * missing it. + */ + + if (left_to_sleep > 0) { + left_to_sleep = + wait_event_interruptible_timeout( + channel->wr_wait, + (!channel->wr_sleepy), + left_to_sleep); + + if (left_to_sleep > 0) /* wr_sleepy deasserted */ + continue; + + if (left_to_sleep < 0) { /* Interrupt */ + mutex_unlock(&channel->wr_mutex); + if (channel->endpoint->fatal_error) + return -EIO; + if (bytes_done) + return bytes_done; + return -EINTR; + } + } + +desperate: + no_time_left = 1; /* We're out of sleeping time. Desperate! */ + + if (bytes_done == 0) { + /* + * Reaching here means that we allow partial return, + * that we've run out of time, and that we have + * nothing to return. + * So tell the FPGA to send anything it has or gets. + */ + + iowrite32(1 | (channel->chan_num << 1) | + (3 << 24) | /* Opcode 3, flush it all! */ + (waiting_bufidx << 12), + channel->endpoint->registers + + fpga_buf_ctrl_reg); + } + + /* + * Reaching here means that we *do* have data in the buffer, + * but the "partial" flag disallows returning less than + * required. And we don't have as much. So loop again, + * which is likely to end up blocking indefinitely until + * enough data has arrived. + */ + } + + mutex_unlock(&channel->wr_mutex); + + if (channel->endpoint->fatal_error) + return -EIO; + + if (rc) + return rc; + + return bytes_done; +} + +/* + * The timeout argument takes values as follows: + * >0 : Flush with timeout + * ==0 : Flush, and wait idefinitely for the flush to complete + * <0 : Autoflush: Flush only if there's a single buffer occupied + */ + +static int xillybus_myflush(struct xilly_channel *channel, long timeout) +{ + int rc; + unsigned long flags; + + int end_offset_plus1; + int bufidx, bufidx_minus1; + int i; + int empty; + int new_rd_host_buf_pos; + + if (channel->endpoint->fatal_error) + return -EIO; + rc = mutex_lock_interruptible(&channel->rd_mutex); + if (rc) + return rc; + + /* + * Don't flush a closed channel. This can happen when the work queued + * autoflush thread fires off after the file has closed. This is not + * an error, just something to dismiss. + */ + + if (!channel->rd_ref_count) + goto done; + + bufidx = channel->rd_host_buf_idx; + + bufidx_minus1 = (bufidx == 0) ? + channel->num_rd_buffers - 1 : + bufidx - 1; + + end_offset_plus1 = channel->rd_host_buf_pos >> + channel->log2_element_size; + + new_rd_host_buf_pos = channel->rd_host_buf_pos - + (end_offset_plus1 << channel->log2_element_size); + + /* Submit the current buffer if it's nonempty */ + if (end_offset_plus1) { + unsigned char *tail = channel->rd_buffers[bufidx]->addr + + (end_offset_plus1 << channel->log2_element_size); + + /* Copy unflushed data, so we can put it in next buffer */ + for (i = 0; i < new_rd_host_buf_pos; i++) + channel->rd_leftovers[i] = *tail++; + + spin_lock_irqsave(&channel->rd_spinlock, flags); + + /* Autoflush only if a single buffer is occupied */ + + if ((timeout < 0) && + (channel->rd_full || + (bufidx_minus1 != channel->rd_fpga_buf_idx))) { + spin_unlock_irqrestore(&channel->rd_spinlock, flags); + /* + * A new work item may be queued by the ISR exactly + * now, since the execution of a work item allows the + * queuing of a new one while it's running. + */ + goto done; + } + + /* The 4th element is never needed for data, so it's a flag */ + channel->rd_leftovers[3] = (new_rd_host_buf_pos != 0); + + /* Set up rd_full to reflect a certain moment's state */ + + if (bufidx == channel->rd_fpga_buf_idx) + channel->rd_full = 1; + spin_unlock_irqrestore(&channel->rd_spinlock, flags); + + if (bufidx >= (channel->num_rd_buffers - 1)) + channel->rd_host_buf_idx = 0; + else + channel->rd_host_buf_idx++; + + dma_sync_single_for_device(channel->endpoint->dev, + channel->rd_buffers[bufidx]->dma_addr, + channel->rd_buf_size, + DMA_TO_DEVICE); + + mutex_lock(&channel->endpoint->register_mutex); + + iowrite32(end_offset_plus1 - 1, + channel->endpoint->registers + fpga_buf_offset_reg); + + iowrite32((channel->chan_num << 1) | /* Channel ID */ + (2 << 24) | /* Opcode 2, submit buffer */ + (bufidx << 12), + channel->endpoint->registers + fpga_buf_ctrl_reg); + + mutex_unlock(&channel->endpoint->register_mutex); + } else if (bufidx == 0) { + bufidx = channel->num_rd_buffers - 1; + } else { + bufidx--; + } + + channel->rd_host_buf_pos = new_rd_host_buf_pos; + + if (timeout < 0) + goto done; /* Autoflush */ + + /* + * bufidx is now the last buffer written to (or equal to + * rd_fpga_buf_idx if buffer was never written to), and + * channel->rd_host_buf_idx the one after it. + * + * If bufidx == channel->rd_fpga_buf_idx we're either empty or full. + */ + + while (1) { /* Loop waiting for draining of buffers */ + spin_lock_irqsave(&channel->rd_spinlock, flags); + + if (bufidx != channel->rd_fpga_buf_idx) + channel->rd_full = 1; /* + * Not really full, + * but needs waiting. + */ + + empty = !channel->rd_full; + + spin_unlock_irqrestore(&channel->rd_spinlock, flags); + + if (empty) + break; + + /* + * Indefinite sleep with mutex taken. With data waiting for + * flushing user should not be surprised if open() for write + * sleeps. + */ + if (timeout == 0) + wait_event_interruptible(channel->rd_wait, + (!channel->rd_full)); + + else if (wait_event_interruptible_timeout( + channel->rd_wait, + (!channel->rd_full), + timeout) == 0) { + dev_warn(channel->endpoint->dev, + "Timed out while flushing. Output data may be lost.\n"); + + rc = -ETIMEDOUT; + break; + } + + if (channel->rd_full) { + rc = -EINTR; + break; + } + } + +done: + mutex_unlock(&channel->rd_mutex); + + if (channel->endpoint->fatal_error) + return -EIO; + + return rc; +} + +static int xillybus_flush(struct file *filp, fl_owner_t id) +{ + if (!(filp->f_mode & FMODE_WRITE)) + return 0; + + return xillybus_myflush(filp->private_data, HZ); /* 1 second timeout */ +} + +static void xillybus_autoflush(struct work_struct *work) +{ + struct delayed_work *workitem = container_of( + work, struct delayed_work, work); + struct xilly_channel *channel = container_of( + workitem, struct xilly_channel, rd_workitem); + int rc; + + rc = xillybus_myflush(channel, -1); + if (rc == -EINTR) + dev_warn(channel->endpoint->dev, + "Autoflush failed because work queue thread got a signal.\n"); + else if (rc) + dev_err(channel->endpoint->dev, + "Autoflush failed under weird circumstances.\n"); +} + +static ssize_t xillybus_write(struct file *filp, const char __user *userbuf, + size_t count, loff_t *f_pos) +{ + ssize_t rc; + unsigned long flags; + int bytes_done = 0; + struct xilly_channel *channel = filp->private_data; + + int full, exhausted; + /* Initializations are there only to silence warnings */ + + int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0; + int end_offset_plus1 = 0; + + if (channel->endpoint->fatal_error) + return -EIO; + + rc = mutex_lock_interruptible(&channel->rd_mutex); + if (rc) + return rc; + + while (1) { + int bytes_to_do = count - bytes_done; + + spin_lock_irqsave(&channel->rd_spinlock, flags); + + full = channel->rd_full; + + if (!full) { + bufidx = channel->rd_host_buf_idx; + bufpos = channel->rd_host_buf_pos; + howmany = channel->rd_buf_size - bufpos; + + /* + * Update rd_host_* to its state after this operation. + * count=0 means committing the buffer immediately, + * which is like flushing, but not necessarily block. + */ + + if ((howmany > bytes_to_do) && + (count || + ((bufpos >> channel->log2_element_size) == 0))) { + bufferdone = 0; + + howmany = bytes_to_do; + channel->rd_host_buf_pos += howmany; + } else { + bufferdone = 1; + + if (count) { + end_offset_plus1 = + channel->rd_buf_size >> + channel->log2_element_size; + channel->rd_host_buf_pos = 0; + } else { + unsigned char *tail; + int i; + + howmany = 0; + + end_offset_plus1 = bufpos >> + channel->log2_element_size; + + channel->rd_host_buf_pos -= + end_offset_plus1 << + channel->log2_element_size; + + tail = channel-> + rd_buffers[bufidx]->addr + + (end_offset_plus1 << + channel->log2_element_size); + + for (i = 0; + i < channel->rd_host_buf_pos; + i++) + channel->rd_leftovers[i] = + *tail++; + } + + if (bufidx == channel->rd_fpga_buf_idx) + channel->rd_full = 1; + + if (bufidx >= (channel->num_rd_buffers - 1)) + channel->rd_host_buf_idx = 0; + else + channel->rd_host_buf_idx++; + } + } + + /* + * Marking our situation after the possible changes above, + * for use after releasing the spinlock. + * + * full = full before change + * exhasted = full after possible change + */ + + exhausted = channel->rd_full; + + spin_unlock_irqrestore(&channel->rd_spinlock, flags); + + if (!full) { /* Go on, now without the spinlock */ + unsigned char *head = + channel->rd_buffers[bufidx]->addr; + int i; + + if ((bufpos == 0) || /* Zero means it's virgin */ + (channel->rd_leftovers[3] != 0)) { + dma_sync_single_for_cpu(channel->endpoint->dev, + channel->rd_buffers[bufidx]->dma_addr, + channel->rd_buf_size, + DMA_TO_DEVICE); + + /* Virgin, but leftovers are due */ + for (i = 0; i < bufpos; i++) + *head++ = channel->rd_leftovers[i]; + + channel->rd_leftovers[3] = 0; /* Clear flag */ + } + + if (copy_from_user( + channel->rd_buffers[bufidx]->addr + bufpos, + userbuf, howmany)) + rc = -EFAULT; + + userbuf += howmany; + bytes_done += howmany; + + if (bufferdone) { + dma_sync_single_for_device(channel->endpoint->dev, + channel->rd_buffers[bufidx]->dma_addr, + channel->rd_buf_size, + DMA_TO_DEVICE); + + mutex_lock(&channel->endpoint->register_mutex); + + iowrite32(end_offset_plus1 - 1, + channel->endpoint->registers + + fpga_buf_offset_reg); + + iowrite32((channel->chan_num << 1) | + (2 << 24) | /* 2 = submit buffer */ + (bufidx << 12), + channel->endpoint->registers + + fpga_buf_ctrl_reg); + + mutex_unlock(&channel->endpoint-> + register_mutex); + + channel->rd_leftovers[3] = + (channel->rd_host_buf_pos != 0); + } + + if (rc) { + mutex_unlock(&channel->rd_mutex); + + if (channel->endpoint->fatal_error) + return -EIO; + + if (!channel->rd_synchronous) + queue_delayed_work( + xillybus_wq, + &channel->rd_workitem, + XILLY_RX_TIMEOUT); + + return rc; + } + } + + if (bytes_done >= count) + break; + + if (!exhausted) + continue; /* If there's more space, just go on */ + + if ((bytes_done > 0) && channel->rd_allow_partial) + break; + + /* + * Indefinite sleep with mutex taken. With data waiting for + * flushing, user should not be surprised if open() for write + * sleeps. + */ + + if (filp->f_flags & O_NONBLOCK) { + rc = -EAGAIN; + break; + } + + if (wait_event_interruptible(channel->rd_wait, + (!channel->rd_full))) { + mutex_unlock(&channel->rd_mutex); + + if (channel->endpoint->fatal_error) + return -EIO; + + if (bytes_done) + return bytes_done; + return -EINTR; + } + } + + mutex_unlock(&channel->rd_mutex); + + if (!channel->rd_synchronous) + queue_delayed_work(xillybus_wq, + &channel->rd_workitem, + XILLY_RX_TIMEOUT); + + if (channel->endpoint->fatal_error) + return -EIO; + + if (rc) + return rc; + + if ((channel->rd_synchronous) && (bytes_done > 0)) { + rc = xillybus_myflush(filp->private_data, 0); /* No timeout */ + + if (rc && (rc != -EINTR)) + return rc; + } + + return bytes_done; +} + +static int xillybus_open(struct inode *inode, struct file *filp) +{ + int rc; + unsigned long flags; + struct xilly_endpoint *endpoint; + struct xilly_channel *channel; + int index; + + rc = xillybus_find_inode(inode, (void **)&endpoint, &index); + if (rc) + return rc; + + if (endpoint->fatal_error) + return -EIO; + + channel = endpoint->channels[1 + index]; + filp->private_data = channel; + + /* + * It gets complicated because: + * 1. We don't want to take a mutex we don't have to + * 2. We don't want to open one direction if the other will fail. + */ + + if ((filp->f_mode & FMODE_READ) && (!channel->num_wr_buffers)) + return -ENODEV; + + if ((filp->f_mode & FMODE_WRITE) && (!channel->num_rd_buffers)) + return -ENODEV; + + if ((filp->f_mode & FMODE_READ) && (filp->f_flags & O_NONBLOCK) && + (channel->wr_synchronous || !channel->wr_allow_partial || + !channel->wr_supports_nonempty)) { + dev_err(endpoint->dev, + "open() failed: O_NONBLOCK not allowed for read on this device\n"); + return -ENODEV; + } + + if ((filp->f_mode & FMODE_WRITE) && (filp->f_flags & O_NONBLOCK) && + (channel->rd_synchronous || !channel->rd_allow_partial)) { + dev_err(endpoint->dev, + "open() failed: O_NONBLOCK not allowed for write on this device\n"); + return -ENODEV; + } + + /* + * Note: open() may block on getting mutexes despite O_NONBLOCK. + * This shouldn't occur normally, since multiple open of the same + * file descriptor is almost always prohibited anyhow + * (*_exclusive_open is normally set in real-life systems). + */ + + if (filp->f_mode & FMODE_READ) { + rc = mutex_lock_interruptible(&channel->wr_mutex); + if (rc) + return rc; + } + + if (filp->f_mode & FMODE_WRITE) { + rc = mutex_lock_interruptible(&channel->rd_mutex); + if (rc) + goto unlock_wr; + } + + if ((filp->f_mode & FMODE_READ) && + (channel->wr_ref_count != 0) && + (channel->wr_exclusive_open)) { + rc = -EBUSY; + goto unlock; + } + + if ((filp->f_mode & FMODE_WRITE) && + (channel->rd_ref_count != 0) && + (channel->rd_exclusive_open)) { + rc = -EBUSY; + goto unlock; + } + + if (filp->f_mode & FMODE_READ) { + if (channel->wr_ref_count == 0) { /* First open of file */ + /* Move the host to first buffer */ + spin_lock_irqsave(&channel->wr_spinlock, flags); + channel->wr_host_buf_idx = 0; + channel->wr_host_buf_pos = 0; + channel->wr_fpga_buf_idx = -1; + channel->wr_empty = 1; + channel->wr_ready = 0; + channel->wr_sleepy = 1; + channel->wr_eof = -1; + channel->wr_hangup = 0; + + spin_unlock_irqrestore(&channel->wr_spinlock, flags); + + iowrite32(1 | (channel->chan_num << 1) | + (4 << 24) | /* Opcode 4, open channel */ + ((channel->wr_synchronous & 1) << 23), + channel->endpoint->registers + + fpga_buf_ctrl_reg); + } + + channel->wr_ref_count++; + } + + if (filp->f_mode & FMODE_WRITE) { + if (channel->rd_ref_count == 0) { /* First open of file */ + /* Move the host to first buffer */ + spin_lock_irqsave(&channel->rd_spinlock, flags); + channel->rd_host_buf_idx = 0; + channel->rd_host_buf_pos = 0; + channel->rd_leftovers[3] = 0; /* No leftovers. */ + channel->rd_fpga_buf_idx = channel->num_rd_buffers - 1; + channel->rd_full = 0; + + spin_unlock_irqrestore(&channel->rd_spinlock, flags); + + iowrite32((channel->chan_num << 1) | + (4 << 24), /* Opcode 4, open channel */ + channel->endpoint->registers + + fpga_buf_ctrl_reg); + } + + channel->rd_ref_count++; + } + +unlock: + if (filp->f_mode & FMODE_WRITE) + mutex_unlock(&channel->rd_mutex); +unlock_wr: + if (filp->f_mode & FMODE_READ) + mutex_unlock(&channel->wr_mutex); + + if (!rc && (!channel->seekable)) + return nonseekable_open(inode, filp); + + return rc; +} + +static int xillybus_release(struct inode *inode, struct file *filp) +{ + unsigned long flags; + struct xilly_channel *channel = filp->private_data; + + int buf_idx; + int eof; + + if (channel->endpoint->fatal_error) + return -EIO; + + if (filp->f_mode & FMODE_WRITE) { + mutex_lock(&channel->rd_mutex); + + channel->rd_ref_count--; + + if (channel->rd_ref_count == 0) { + /* + * We rely on the kernel calling flush() + * before we get here. + */ + + iowrite32((channel->chan_num << 1) | /* Channel ID */ + (5 << 24), /* Opcode 5, close channel */ + channel->endpoint->registers + + fpga_buf_ctrl_reg); + } + mutex_unlock(&channel->rd_mutex); + } + + if (filp->f_mode & FMODE_READ) { + mutex_lock(&channel->wr_mutex); + + channel->wr_ref_count--; + + if (channel->wr_ref_count == 0) { + iowrite32(1 | (channel->chan_num << 1) | + (5 << 24), /* Opcode 5, close channel */ + channel->endpoint->registers + + fpga_buf_ctrl_reg); + + /* + * This is crazily cautious: We make sure that not + * only that we got an EOF (be it because we closed + * the channel or because of a user's EOF), but verify + * that it's one beyond the last buffer arrived, so + * we have no leftover buffers pending before wrapping + * up (which can only happen in asynchronous channels, + * BTW) + */ + + while (1) { + spin_lock_irqsave(&channel->wr_spinlock, + flags); + buf_idx = channel->wr_fpga_buf_idx; + eof = channel->wr_eof; + channel->wr_sleepy = 1; + spin_unlock_irqrestore(&channel->wr_spinlock, + flags); + + /* + * Check if eof points at the buffer after + * the last one the FPGA submitted. Note that + * no EOF is marked by negative eof. + */ + + buf_idx++; + if (buf_idx == channel->num_wr_buffers) + buf_idx = 0; + + if (buf_idx == eof) + break; + + /* + * Steal extra 100 ms if awaken by interrupt. + * This is a simple workaround for an + * interrupt pending when entering, which would + * otherwise result in declaring the hardware + * non-responsive. + */ + + if (wait_event_interruptible( + channel->wr_wait, + (!channel->wr_sleepy))) + msleep(100); + + if (channel->wr_sleepy) { + mutex_unlock(&channel->wr_mutex); + dev_warn(channel->endpoint->dev, + "Hardware failed to respond to close command, therefore left in messy state.\n"); + return -EINTR; + } + } + } + + mutex_unlock(&channel->wr_mutex); + } + + return 0; +} + +static loff_t xillybus_llseek(struct file *filp, loff_t offset, int whence) +{ + struct xilly_channel *channel = filp->private_data; + loff_t pos = filp->f_pos; + int rc = 0; + + /* + * Take both mutexes not allowing interrupts, since it seems like + * common applications don't expect an -EINTR here. Besides, multiple + * access to a single file descriptor on seekable devices is a mess + * anyhow. + */ + + if (channel->endpoint->fatal_error) + return -EIO; + + mutex_lock(&channel->wr_mutex); + mutex_lock(&channel->rd_mutex); + + switch (whence) { + case SEEK_SET: + pos = offset; + break; + case SEEK_CUR: + pos += offset; + break; + case SEEK_END: + pos = offset; /* Going to the end => to the beginning */ + break; + default: + rc = -EINVAL; + goto end; + } + + /* In any case, we must finish on an element boundary */ + if (pos & ((1 << channel->log2_element_size) - 1)) { + rc = -EINVAL; + goto end; + } + + mutex_lock(&channel->endpoint->register_mutex); + + iowrite32(pos >> channel->log2_element_size, + channel->endpoint->registers + fpga_buf_offset_reg); + + iowrite32((channel->chan_num << 1) | + (6 << 24), /* Opcode 6, set address */ + channel->endpoint->registers + fpga_buf_ctrl_reg); + + mutex_unlock(&channel->endpoint->register_mutex); + +end: + mutex_unlock(&channel->rd_mutex); + mutex_unlock(&channel->wr_mutex); + + if (rc) /* Return error after releasing mutexes */ + return rc; + + filp->f_pos = pos; + + /* + * Since seekable devices are allowed only when the channel is + * synchronous, we assume that there is no data pending in either + * direction (which holds true as long as no concurrent access on the + * file descriptor takes place). + * The only thing we may need to throw away is leftovers from partial + * write() flush. + */ + + channel->rd_leftovers[3] = 0; + + return pos; +} + +static __poll_t xillybus_poll(struct file *filp, poll_table *wait) +{ + struct xilly_channel *channel = filp->private_data; + __poll_t mask = 0; + unsigned long flags; + + poll_wait(filp, &channel->endpoint->ep_wait, wait); + + /* + * poll() won't play ball regarding read() channels which + * aren't asynchronous and support the nonempty message. Allowing + * that will create situations where data has been delivered at + * the FPGA, and users expecting select() to wake up, which it may + * not. + */ + + if (!channel->wr_synchronous && channel->wr_supports_nonempty) { + poll_wait(filp, &channel->wr_wait, wait); + poll_wait(filp, &channel->wr_ready_wait, wait); + + spin_lock_irqsave(&channel->wr_spinlock, flags); + if (!channel->wr_empty || channel->wr_ready) + mask |= EPOLLIN | EPOLLRDNORM; + + if (channel->wr_hangup) + /* + * Not EPOLLHUP, because its behavior is in the + * mist, and EPOLLIN does what we want: Wake up + * the read file descriptor so it sees EOF. + */ + mask |= EPOLLIN | EPOLLRDNORM; + spin_unlock_irqrestore(&channel->wr_spinlock, flags); + } + + /* + * If partial data write is disallowed on a write() channel, + * it's pointless to ever signal OK to write, because is could + * block despite some space being available. + */ + + if (channel->rd_allow_partial) { + poll_wait(filp, &channel->rd_wait, wait); + + spin_lock_irqsave(&channel->rd_spinlock, flags); + if (!channel->rd_full) + mask |= EPOLLOUT | EPOLLWRNORM; + spin_unlock_irqrestore(&channel->rd_spinlock, flags); + } + + if (channel->endpoint->fatal_error) + mask |= EPOLLERR; + + return mask; +} + +static const struct file_operations xillybus_fops = { + .owner = THIS_MODULE, + .read = xillybus_read, + .write = xillybus_write, + .open = xillybus_open, + .flush = xillybus_flush, + .release = xillybus_release, + .llseek = xillybus_llseek, + .poll = xillybus_poll, +}; + +struct xilly_endpoint *xillybus_init_endpoint(struct device *dev) +{ + struct xilly_endpoint *endpoint; + + endpoint = devm_kzalloc(dev, sizeof(*endpoint), GFP_KERNEL); + if (!endpoint) + return NULL; + + endpoint->dev = dev; + endpoint->msg_counter = 0x0b; + endpoint->failed_messages = 0; + endpoint->fatal_error = 0; + + init_waitqueue_head(&endpoint->ep_wait); + mutex_init(&endpoint->register_mutex); + + return endpoint; +} +EXPORT_SYMBOL(xillybus_init_endpoint); + +static int xilly_quiesce(struct xilly_endpoint *endpoint) +{ + long t; + + endpoint->idtlen = -1; + + iowrite32((u32) (endpoint->dma_using_dac & 0x0001), + endpoint->registers + fpga_dma_control_reg); + + t = wait_event_interruptible_timeout(endpoint->ep_wait, + (endpoint->idtlen >= 0), + XILLY_TIMEOUT); + if (t <= 0) { + dev_err(endpoint->dev, + "Failed to quiesce the device on exit.\n"); + return -ENODEV; + } + return 0; +} + +int xillybus_endpoint_discovery(struct xilly_endpoint *endpoint) +{ + int rc; + long t; + + void *bootstrap_resources; + int idtbuffersize = (1 << PAGE_SHIFT); + struct device *dev = endpoint->dev; + + /* + * The bogus IDT is used during bootstrap for allocating the initial + * message buffer, and then the message buffer and space for the IDT + * itself. The initial message buffer is of a single page's size, but + * it's soon replaced with a more modest one (and memory is freed). + */ + + unsigned char bogus_idt[8] = { 1, 224, (PAGE_SHIFT)-2, 0, + 3, 192, PAGE_SHIFT, 0 }; + struct xilly_idt_handle idt_handle; + + /* + * Writing the value 0x00000001 to Endianness register signals which + * endianness this processor is using, so the FPGA can swap words as + * necessary. + */ + + iowrite32(1, endpoint->registers + fpga_endian_reg); + + /* Bootstrap phase I: Allocate temporary message buffer */ + + bootstrap_resources = devres_open_group(dev, NULL, GFP_KERNEL); + if (!bootstrap_resources) + return -ENOMEM; + + endpoint->num_channels = 0; + + rc = xilly_setupchannels(endpoint, bogus_idt, 1); + if (rc) + return rc; + + /* Clear the message subsystem (and counter in particular) */ + iowrite32(0x04, endpoint->registers + fpga_msg_ctrl_reg); + + endpoint->idtlen = -1; + + /* + * Set DMA 32/64 bit mode, quiesce the device (?!) and get IDT + * buffer size. + */ + iowrite32((u32) (endpoint->dma_using_dac & 0x0001), + endpoint->registers + fpga_dma_control_reg); + + t = wait_event_interruptible_timeout(endpoint->ep_wait, + (endpoint->idtlen >= 0), + XILLY_TIMEOUT); + if (t <= 0) { + dev_err(endpoint->dev, "No response from FPGA. Aborting.\n"); + return -ENODEV; + } + + /* Enable DMA */ + iowrite32((u32) (0x0002 | (endpoint->dma_using_dac & 0x0001)), + endpoint->registers + fpga_dma_control_reg); + + /* Bootstrap phase II: Allocate buffer for IDT and obtain it */ + while (endpoint->idtlen >= idtbuffersize) { + idtbuffersize *= 2; + bogus_idt[6]++; + } + + endpoint->num_channels = 1; + + rc = xilly_setupchannels(endpoint, bogus_idt, 2); + if (rc) + goto failed_idt; + + rc = xilly_obtain_idt(endpoint); + if (rc) + goto failed_idt; + + rc = xilly_scan_idt(endpoint, &idt_handle); + if (rc) + goto failed_idt; + + devres_close_group(dev, bootstrap_resources); + + /* Bootstrap phase III: Allocate buffers according to IDT */ + + rc = xilly_setupchannels(endpoint, + idt_handle.chandesc, + idt_handle.entries); + if (rc) + goto failed_idt; + + rc = xillybus_init_chrdev(dev, &xillybus_fops, + endpoint->owner, endpoint, + idt_handle.names, + idt_handle.names_len, + endpoint->num_channels, + xillyname, false); + + if (rc) + goto failed_idt; + + devres_release_group(dev, bootstrap_resources); + + return 0; + +failed_idt: + xilly_quiesce(endpoint); + flush_workqueue(xillybus_wq); + + return rc; +} +EXPORT_SYMBOL(xillybus_endpoint_discovery); + +void xillybus_endpoint_remove(struct xilly_endpoint *endpoint) +{ + xillybus_cleanup_chrdev(endpoint, endpoint->dev); + + xilly_quiesce(endpoint); + + /* + * Flushing is done upon endpoint release to prevent access to memory + * just about to be released. This makes the quiesce complete. + */ + flush_workqueue(xillybus_wq); +} +EXPORT_SYMBOL(xillybus_endpoint_remove); + +static int __init xillybus_init(void) +{ + xillybus_wq = alloc_workqueue(xillyname, 0, 0); + if (!xillybus_wq) + return -ENOMEM; + + return 0; +} + +static void __exit xillybus_exit(void) +{ + /* flush_workqueue() was called for each endpoint released */ + destroy_workqueue(xillybus_wq); +} + +module_init(xillybus_init); +module_exit(xillybus_exit); diff --git a/drivers/char/xillybus/xillybus_of.c b/drivers/char/xillybus/xillybus_of.c new file mode 100644 index 0000000000..e5372e45d2 --- /dev/null +++ b/drivers/char/xillybus/xillybus_of.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/drivers/misc/xillybus_of.c + * + * Copyright 2011 Xillybus Ltd, http://xillybus.com + * + * Driver for the Xillybus FPGA/host framework using Open Firmware. + */ + +#include <linux/module.h> +#include <linux/device.h> +#include <linux/slab.h> +#include <linux/platform_device.h> +#include <linux/of.h> +#include <linux/err.h> +#include "xillybus.h" + +MODULE_DESCRIPTION("Xillybus driver for Open Firmware"); +MODULE_AUTHOR("Eli Billauer, Xillybus Ltd."); +MODULE_ALIAS("xillybus_of"); +MODULE_LICENSE("GPL v2"); + +static const char xillyname[] = "xillybus_of"; + +/* Match table for of_platform binding */ +static const struct of_device_id xillybus_of_match[] = { + { .compatible = "xillybus,xillybus-1.00.a", }, + { .compatible = "xlnx,xillybus-1.00.a", }, /* Deprecated */ + {} +}; + +MODULE_DEVICE_TABLE(of, xillybus_of_match); + +static int xilly_drv_probe(struct platform_device *op) +{ + struct device *dev = &op->dev; + struct xilly_endpoint *endpoint; + int rc; + int irq; + + endpoint = xillybus_init_endpoint(dev); + + if (!endpoint) + return -ENOMEM; + + dev_set_drvdata(dev, endpoint); + + endpoint->owner = THIS_MODULE; + + endpoint->registers = devm_platform_ioremap_resource(op, 0); + if (IS_ERR(endpoint->registers)) + return PTR_ERR(endpoint->registers); + + irq = platform_get_irq(op, 0); + + rc = devm_request_irq(dev, irq, xillybus_isr, 0, xillyname, endpoint); + + if (rc) { + dev_err(endpoint->dev, + "Failed to register IRQ handler. Aborting.\n"); + return -ENODEV; + } + + return xillybus_endpoint_discovery(endpoint); +} + +static int xilly_drv_remove(struct platform_device *op) +{ + struct device *dev = &op->dev; + struct xilly_endpoint *endpoint = dev_get_drvdata(dev); + + xillybus_endpoint_remove(endpoint); + + return 0; +} + +static struct platform_driver xillybus_platform_driver = { + .probe = xilly_drv_probe, + .remove = xilly_drv_remove, + .driver = { + .name = xillyname, + .of_match_table = xillybus_of_match, + }, +}; + +module_platform_driver(xillybus_platform_driver); diff --git a/drivers/char/xillybus/xillybus_pcie.c b/drivers/char/xillybus/xillybus_pcie.c new file mode 100644 index 0000000000..9858711e3e --- /dev/null +++ b/drivers/char/xillybus/xillybus_pcie.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/drivers/misc/xillybus_pcie.c + * + * Copyright 2011 Xillybus Ltd, http://xillybus.com + * + * Driver for the Xillybus FPGA/host framework using PCI Express. + */ + +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/slab.h> +#include "xillybus.h" + +MODULE_DESCRIPTION("Xillybus driver for PCIe"); +MODULE_AUTHOR("Eli Billauer, Xillybus Ltd."); +MODULE_ALIAS("xillybus_pcie"); +MODULE_LICENSE("GPL v2"); + +#define PCI_DEVICE_ID_XILLYBUS 0xebeb + +#define PCI_VENDOR_ID_ACTEL 0x11aa +#define PCI_VENDOR_ID_LATTICE 0x1204 + +static const char xillyname[] = "xillybus_pcie"; + +static const struct pci_device_id xillyids[] = { + {PCI_DEVICE(PCI_VENDOR_ID_XILINX, PCI_DEVICE_ID_XILLYBUS)}, + {PCI_DEVICE(PCI_VENDOR_ID_ALTERA, PCI_DEVICE_ID_XILLYBUS)}, + {PCI_DEVICE(PCI_VENDOR_ID_ACTEL, PCI_DEVICE_ID_XILLYBUS)}, + {PCI_DEVICE(PCI_VENDOR_ID_LATTICE, PCI_DEVICE_ID_XILLYBUS)}, + { /* End: all zeroes */ } +}; + +static int xilly_probe(struct pci_dev *pdev, + const struct pci_device_id *ent) +{ + struct xilly_endpoint *endpoint; + int rc; + + endpoint = xillybus_init_endpoint(&pdev->dev); + + if (!endpoint) + return -ENOMEM; + + pci_set_drvdata(pdev, endpoint); + + endpoint->owner = THIS_MODULE; + + rc = pcim_enable_device(pdev); + if (rc) { + dev_err(endpoint->dev, + "pcim_enable_device() failed. Aborting.\n"); + return rc; + } + + /* L0s has caused packet drops. No power saving, thank you. */ + + pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S); + + if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { + dev_err(endpoint->dev, + "Incorrect BAR configuration. Aborting.\n"); + return -ENODEV; + } + + rc = pcim_iomap_regions(pdev, 0x01, xillyname); + if (rc) { + dev_err(endpoint->dev, + "pcim_iomap_regions() failed. Aborting.\n"); + return rc; + } + + endpoint->registers = pcim_iomap_table(pdev)[0]; + + pci_set_master(pdev); + + /* Set up a single MSI interrupt */ + if (pci_enable_msi(pdev)) { + dev_err(endpoint->dev, + "Failed to enable MSI interrupts. Aborting.\n"); + return -ENODEV; + } + rc = devm_request_irq(&pdev->dev, pdev->irq, xillybus_isr, 0, + xillyname, endpoint); + if (rc) { + dev_err(endpoint->dev, + "Failed to register MSI handler. Aborting.\n"); + return -ENODEV; + } + + /* + * Some (old and buggy?) hardware drops 64-bit addressed PCIe packets, + * even when the PCIe driver claims that a 64-bit mask is OK. On the + * other hand, on some architectures, 64-bit addressing is mandatory. + * So go for the 64-bit mask only when failing is the other option. + */ + + if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(32))) { + endpoint->dma_using_dac = 0; + } else if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64))) { + endpoint->dma_using_dac = 1; + } else { + dev_err(endpoint->dev, "Failed to set DMA mask. Aborting.\n"); + return -ENODEV; + } + + return xillybus_endpoint_discovery(endpoint); +} + +static void xilly_remove(struct pci_dev *pdev) +{ + struct xilly_endpoint *endpoint = pci_get_drvdata(pdev); + + xillybus_endpoint_remove(endpoint); +} + +MODULE_DEVICE_TABLE(pci, xillyids); + +static struct pci_driver xillybus_driver = { + .name = xillyname, + .id_table = xillyids, + .probe = xilly_probe, + .remove = xilly_remove, +}; + +module_pci_driver(xillybus_driver); diff --git a/drivers/char/xillybus/xillyusb.c b/drivers/char/xillybus/xillyusb.c new file mode 100644 index 0000000000..5a5afa14ca --- /dev/null +++ b/drivers/char/xillybus/xillyusb.c @@ -0,0 +1,2277 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2020 Xillybus Ltd, http://xillybus.com + * + * Driver for the XillyUSB FPGA/host framework. + * + * This driver interfaces with a special IP core in an FPGA, setting up + * a pipe between a hardware FIFO in the programmable logic and a device + * file in the host. The number of such pipes and their attributes are + * set up on the logic. This driver detects these automatically and + * creates the device files accordingly. + */ + +#include <linux/types.h> +#include <linux/slab.h> +#include <linux/list.h> +#include <linux/device.h> +#include <linux/module.h> +#include <asm/byteorder.h> +#include <linux/io.h> +#include <linux/interrupt.h> +#include <linux/sched.h> +#include <linux/fs.h> +#include <linux/spinlock.h> +#include <linux/mutex.h> +#include <linux/workqueue.h> +#include <linux/crc32.h> +#include <linux/poll.h> +#include <linux/delay.h> +#include <linux/usb.h> + +#include "xillybus_class.h" + +MODULE_DESCRIPTION("Driver for XillyUSB FPGA IP Core"); +MODULE_AUTHOR("Eli Billauer, Xillybus Ltd."); +MODULE_ALIAS("xillyusb"); +MODULE_LICENSE("GPL v2"); + +#define XILLY_RX_TIMEOUT (10 * HZ / 1000) +#define XILLY_RESPONSE_TIMEOUT (500 * HZ / 1000) + +#define BUF_SIZE_ORDER 4 +#define BUFNUM 8 +#define LOG2_IDT_FIFO_SIZE 16 +#define LOG2_INITIAL_FIFO_BUF_SIZE 16 + +#define MSG_EP_NUM 1 +#define IN_EP_NUM 1 + +static const char xillyname[] = "xillyusb"; + +static unsigned int fifo_buf_order; + +#define USB_VENDOR_ID_XILINX 0x03fd +#define USB_VENDOR_ID_ALTERA 0x09fb + +#define USB_PRODUCT_ID_XILLYUSB 0xebbe + +static const struct usb_device_id xillyusb_table[] = { + { USB_DEVICE(USB_VENDOR_ID_XILINX, USB_PRODUCT_ID_XILLYUSB) }, + { USB_DEVICE(USB_VENDOR_ID_ALTERA, USB_PRODUCT_ID_XILLYUSB) }, + { } +}; + +MODULE_DEVICE_TABLE(usb, xillyusb_table); + +struct xillyusb_dev; + +struct xillyfifo { + unsigned int bufsize; /* In bytes, always a power of 2 */ + unsigned int bufnum; + unsigned int size; /* Lazy: Equals bufsize * bufnum */ + unsigned int buf_order; + + int fill; /* Number of bytes in the FIFO */ + spinlock_t lock; + wait_queue_head_t waitq; + + unsigned int readpos; + unsigned int readbuf; + unsigned int writepos; + unsigned int writebuf; + char **mem; +}; + +struct xillyusb_channel; + +struct xillyusb_endpoint { + struct xillyusb_dev *xdev; + + struct mutex ep_mutex; /* serialize operations on endpoint */ + + struct list_head buffers; + struct list_head filled_buffers; + spinlock_t buffers_lock; /* protect these two lists */ + + unsigned int order; + unsigned int buffer_size; + + unsigned int fill_mask; + + int outstanding_urbs; + + struct usb_anchor anchor; + + struct xillyfifo fifo; + + struct work_struct workitem; + + bool shutting_down; + bool drained; + bool wake_on_drain; + + u8 ep_num; +}; + +struct xillyusb_channel { + struct xillyusb_dev *xdev; + + struct xillyfifo *in_fifo; + struct xillyusb_endpoint *out_ep; + struct mutex lock; /* protect @out_ep, @in_fifo, bit fields below */ + + struct mutex in_mutex; /* serialize fops on FPGA to host stream */ + struct mutex out_mutex; /* serialize fops on host to FPGA stream */ + wait_queue_head_t flushq; + + int chan_idx; + + u32 in_consumed_bytes; + u32 in_current_checkpoint; + u32 out_bytes; + + unsigned int in_log2_element_size; + unsigned int out_log2_element_size; + unsigned int in_log2_fifo_size; + unsigned int out_log2_fifo_size; + + unsigned int read_data_ok; /* EOF not arrived (yet) */ + unsigned int poll_used; + unsigned int flushing; + unsigned int flushed; + unsigned int canceled; + + /* Bit fields protected by @lock except for initialization */ + unsigned readable:1; + unsigned writable:1; + unsigned open_for_read:1; + unsigned open_for_write:1; + unsigned in_synchronous:1; + unsigned out_synchronous:1; + unsigned in_seekable:1; + unsigned out_seekable:1; +}; + +struct xillybuffer { + struct list_head entry; + struct xillyusb_endpoint *ep; + void *buf; + unsigned int len; +}; + +struct xillyusb_dev { + struct xillyusb_channel *channels; + + struct usb_device *udev; + struct device *dev; /* For dev_err() and such */ + struct kref kref; + struct workqueue_struct *workq; + + int error; + spinlock_t error_lock; /* protect @error */ + struct work_struct wakeup_workitem; + + int num_channels; + + struct xillyusb_endpoint *msg_ep; + struct xillyusb_endpoint *in_ep; + + struct mutex msg_mutex; /* serialize opcode transmission */ + int in_bytes_left; + int leftover_chan_num; + unsigned int in_counter; + struct mutex process_in_mutex; /* synchronize wakeup_all() */ +}; + +/* + * kref_mutex is used in xillyusb_open() to prevent the xillyusb_dev + * struct from being freed during the gap between being found by + * xillybus_find_inode() and having its reference count incremented. + */ + +static DEFINE_MUTEX(kref_mutex); + +/* FPGA to host opcodes */ +enum { + OPCODE_DATA = 0, + OPCODE_QUIESCE_ACK = 1, + OPCODE_EOF = 2, + OPCODE_REACHED_CHECKPOINT = 3, + OPCODE_CANCELED_CHECKPOINT = 4, +}; + +/* Host to FPGA opcodes */ +enum { + OPCODE_QUIESCE = 0, + OPCODE_REQ_IDT = 1, + OPCODE_SET_CHECKPOINT = 2, + OPCODE_CLOSE = 3, + OPCODE_SET_PUSH = 4, + OPCODE_UPDATE_PUSH = 5, + OPCODE_CANCEL_CHECKPOINT = 6, + OPCODE_SET_ADDR = 7, +}; + +/* + * fifo_write() and fifo_read() are NOT reentrant (i.e. concurrent multiple + * calls to each on the same FIFO is not allowed) however it's OK to have + * threads calling each of the two functions once on the same FIFO, and + * at the same time. + */ + +static int fifo_write(struct xillyfifo *fifo, + const void *data, unsigned int len, + int (*copier)(void *, const void *, int)) +{ + unsigned int done = 0; + unsigned int todo = len; + unsigned int nmax; + unsigned int writepos = fifo->writepos; + unsigned int writebuf = fifo->writebuf; + unsigned long flags; + int rc; + + nmax = fifo->size - READ_ONCE(fifo->fill); + + while (1) { + unsigned int nrail = fifo->bufsize - writepos; + unsigned int n = min(todo, nmax); + + if (n == 0) { + spin_lock_irqsave(&fifo->lock, flags); + fifo->fill += done; + spin_unlock_irqrestore(&fifo->lock, flags); + + fifo->writepos = writepos; + fifo->writebuf = writebuf; + + return done; + } + + if (n > nrail) + n = nrail; + + rc = (*copier)(fifo->mem[writebuf] + writepos, data + done, n); + + if (rc) + return rc; + + done += n; + todo -= n; + + writepos += n; + nmax -= n; + + if (writepos == fifo->bufsize) { + writepos = 0; + writebuf++; + + if (writebuf == fifo->bufnum) + writebuf = 0; + } + } +} + +static int fifo_read(struct xillyfifo *fifo, + void *data, unsigned int len, + int (*copier)(void *, const void *, int)) +{ + unsigned int done = 0; + unsigned int todo = len; + unsigned int fill; + unsigned int readpos = fifo->readpos; + unsigned int readbuf = fifo->readbuf; + unsigned long flags; + int rc; + + /* + * The spinlock here is necessary, because otherwise fifo->fill + * could have been increased by fifo_write() after writing data + * to the buffer, but this data would potentially not have been + * visible on this thread at the time the updated fifo->fill was. + * That could lead to reading invalid data. + */ + + spin_lock_irqsave(&fifo->lock, flags); + fill = fifo->fill; + spin_unlock_irqrestore(&fifo->lock, flags); + + while (1) { + unsigned int nrail = fifo->bufsize - readpos; + unsigned int n = min(todo, fill); + + if (n == 0) { + spin_lock_irqsave(&fifo->lock, flags); + fifo->fill -= done; + spin_unlock_irqrestore(&fifo->lock, flags); + + fifo->readpos = readpos; + fifo->readbuf = readbuf; + + return done; + } + + if (n > nrail) + n = nrail; + + rc = (*copier)(data + done, fifo->mem[readbuf] + readpos, n); + + if (rc) + return rc; + + done += n; + todo -= n; + + readpos += n; + fill -= n; + + if (readpos == fifo->bufsize) { + readpos = 0; + readbuf++; + + if (readbuf == fifo->bufnum) + readbuf = 0; + } + } +} + +/* + * These three wrapper functions are used as the @copier argument to + * fifo_write() and fifo_read(), so that they can work directly with + * user memory as well. + */ + +static int xilly_copy_from_user(void *dst, const void *src, int n) +{ + if (copy_from_user(dst, (const void __user *)src, n)) + return -EFAULT; + + return 0; +} + +static int xilly_copy_to_user(void *dst, const void *src, int n) +{ + if (copy_to_user((void __user *)dst, src, n)) + return -EFAULT; + + return 0; +} + +static int xilly_memcpy(void *dst, const void *src, int n) +{ + memcpy(dst, src, n); + + return 0; +} + +static int fifo_init(struct xillyfifo *fifo, + unsigned int log2_size) +{ + unsigned int log2_bufnum; + unsigned int buf_order; + int i; + + unsigned int log2_fifo_buf_size; + +retry: + log2_fifo_buf_size = fifo_buf_order + PAGE_SHIFT; + + if (log2_size > log2_fifo_buf_size) { + log2_bufnum = log2_size - log2_fifo_buf_size; + buf_order = fifo_buf_order; + fifo->bufsize = 1 << log2_fifo_buf_size; + } else { + log2_bufnum = 0; + buf_order = (log2_size > PAGE_SHIFT) ? + log2_size - PAGE_SHIFT : 0; + fifo->bufsize = 1 << log2_size; + } + + fifo->bufnum = 1 << log2_bufnum; + fifo->size = fifo->bufnum * fifo->bufsize; + fifo->buf_order = buf_order; + + fifo->mem = kmalloc_array(fifo->bufnum, sizeof(void *), GFP_KERNEL); + + if (!fifo->mem) + return -ENOMEM; + + for (i = 0; i < fifo->bufnum; i++) { + fifo->mem[i] = (void *) + __get_free_pages(GFP_KERNEL, buf_order); + + if (!fifo->mem[i]) + goto memfail; + } + + fifo->fill = 0; + fifo->readpos = 0; + fifo->readbuf = 0; + fifo->writepos = 0; + fifo->writebuf = 0; + spin_lock_init(&fifo->lock); + init_waitqueue_head(&fifo->waitq); + return 0; + +memfail: + for (i--; i >= 0; i--) + free_pages((unsigned long)fifo->mem[i], buf_order); + + kfree(fifo->mem); + fifo->mem = NULL; + + if (fifo_buf_order) { + fifo_buf_order--; + goto retry; + } else { + return -ENOMEM; + } +} + +static void fifo_mem_release(struct xillyfifo *fifo) +{ + int i; + + if (!fifo->mem) + return; + + for (i = 0; i < fifo->bufnum; i++) + free_pages((unsigned long)fifo->mem[i], fifo->buf_order); + + kfree(fifo->mem); +} + +/* + * When endpoint_quiesce() returns, the endpoint has no URBs submitted, + * won't accept any new URB submissions, and its related work item doesn't + * and won't run anymore. + */ + +static void endpoint_quiesce(struct xillyusb_endpoint *ep) +{ + mutex_lock(&ep->ep_mutex); + ep->shutting_down = true; + mutex_unlock(&ep->ep_mutex); + + usb_kill_anchored_urbs(&ep->anchor); + cancel_work_sync(&ep->workitem); +} + +/* + * Note that endpoint_dealloc() also frees fifo memory (if allocated), even + * though endpoint_alloc doesn't allocate that memory. + */ + +static void endpoint_dealloc(struct xillyusb_endpoint *ep) +{ + struct list_head *this, *next; + + fifo_mem_release(&ep->fifo); + + /* Join @filled_buffers with @buffers to free these entries too */ + list_splice(&ep->filled_buffers, &ep->buffers); + + list_for_each_safe(this, next, &ep->buffers) { + struct xillybuffer *xb = + list_entry(this, struct xillybuffer, entry); + + free_pages((unsigned long)xb->buf, ep->order); + kfree(xb); + } + + kfree(ep); +} + +static struct xillyusb_endpoint +*endpoint_alloc(struct xillyusb_dev *xdev, + u8 ep_num, + void (*work)(struct work_struct *), + unsigned int order, + int bufnum) +{ + int i; + + struct xillyusb_endpoint *ep; + + ep = kzalloc(sizeof(*ep), GFP_KERNEL); + + if (!ep) + return NULL; + + INIT_LIST_HEAD(&ep->buffers); + INIT_LIST_HEAD(&ep->filled_buffers); + + spin_lock_init(&ep->buffers_lock); + mutex_init(&ep->ep_mutex); + + init_usb_anchor(&ep->anchor); + INIT_WORK(&ep->workitem, work); + + ep->order = order; + ep->buffer_size = 1 << (PAGE_SHIFT + order); + ep->outstanding_urbs = 0; + ep->drained = true; + ep->wake_on_drain = false; + ep->xdev = xdev; + ep->ep_num = ep_num; + ep->shutting_down = false; + + for (i = 0; i < bufnum; i++) { + struct xillybuffer *xb; + unsigned long addr; + + xb = kzalloc(sizeof(*xb), GFP_KERNEL); + + if (!xb) { + endpoint_dealloc(ep); + return NULL; + } + + addr = __get_free_pages(GFP_KERNEL, order); + + if (!addr) { + kfree(xb); + endpoint_dealloc(ep); + return NULL; + } + + xb->buf = (void *)addr; + xb->ep = ep; + list_add_tail(&xb->entry, &ep->buffers); + } + return ep; +} + +static void cleanup_dev(struct kref *kref) +{ + struct xillyusb_dev *xdev = + container_of(kref, struct xillyusb_dev, kref); + + if (xdev->in_ep) + endpoint_dealloc(xdev->in_ep); + + if (xdev->msg_ep) + endpoint_dealloc(xdev->msg_ep); + + if (xdev->workq) + destroy_workqueue(xdev->workq); + + usb_put_dev(xdev->udev); + kfree(xdev->channels); /* Argument may be NULL, and that's fine */ + kfree(xdev); +} + +/* + * @process_in_mutex is taken to ensure that bulk_in_work() won't call + * process_bulk_in() after wakeup_all()'s execution: The latter zeroes all + * @read_data_ok entries, which will make process_bulk_in() report false + * errors if executed. The mechanism relies on that xdev->error is assigned + * a non-zero value by report_io_error() prior to queueing wakeup_all(), + * which prevents bulk_in_work() from calling process_bulk_in(). + * + * The fact that wakeup_all() and bulk_in_work() are queued on the same + * workqueue makes their concurrent execution very unlikely, however the + * kernel's API doesn't seem to ensure this strictly. + */ + +static void wakeup_all(struct work_struct *work) +{ + int i; + struct xillyusb_dev *xdev = container_of(work, struct xillyusb_dev, + wakeup_workitem); + + mutex_lock(&xdev->process_in_mutex); + + for (i = 0; i < xdev->num_channels; i++) { + struct xillyusb_channel *chan = &xdev->channels[i]; + + mutex_lock(&chan->lock); + + if (chan->in_fifo) { + /* + * Fake an EOF: Even if such arrives, it won't be + * processed. + */ + chan->read_data_ok = 0; + wake_up_interruptible(&chan->in_fifo->waitq); + } + + if (chan->out_ep) + wake_up_interruptible(&chan->out_ep->fifo.waitq); + + mutex_unlock(&chan->lock); + + wake_up_interruptible(&chan->flushq); + } + + mutex_unlock(&xdev->process_in_mutex); + + wake_up_interruptible(&xdev->msg_ep->fifo.waitq); + + kref_put(&xdev->kref, cleanup_dev); +} + +static void report_io_error(struct xillyusb_dev *xdev, + int errcode) +{ + unsigned long flags; + bool do_once = false; + + spin_lock_irqsave(&xdev->error_lock, flags); + if (!xdev->error) { + xdev->error = errcode; + do_once = true; + } + spin_unlock_irqrestore(&xdev->error_lock, flags); + + if (do_once) { + kref_get(&xdev->kref); /* xdev is used by work item */ + queue_work(xdev->workq, &xdev->wakeup_workitem); + } +} + +/* + * safely_assign_in_fifo() changes the value of chan->in_fifo and ensures + * the previous pointer is never used after its return. + */ + +static void safely_assign_in_fifo(struct xillyusb_channel *chan, + struct xillyfifo *fifo) +{ + mutex_lock(&chan->lock); + chan->in_fifo = fifo; + mutex_unlock(&chan->lock); + + flush_work(&chan->xdev->in_ep->workitem); +} + +static void bulk_in_completer(struct urb *urb) +{ + struct xillybuffer *xb = urb->context; + struct xillyusb_endpoint *ep = xb->ep; + unsigned long flags; + + if (urb->status) { + if (!(urb->status == -ENOENT || + urb->status == -ECONNRESET || + urb->status == -ESHUTDOWN)) + report_io_error(ep->xdev, -EIO); + + spin_lock_irqsave(&ep->buffers_lock, flags); + list_add_tail(&xb->entry, &ep->buffers); + ep->outstanding_urbs--; + spin_unlock_irqrestore(&ep->buffers_lock, flags); + + return; + } + + xb->len = urb->actual_length; + + spin_lock_irqsave(&ep->buffers_lock, flags); + list_add_tail(&xb->entry, &ep->filled_buffers); + spin_unlock_irqrestore(&ep->buffers_lock, flags); + + if (!ep->shutting_down) + queue_work(ep->xdev->workq, &ep->workitem); +} + +static void bulk_out_completer(struct urb *urb) +{ + struct xillybuffer *xb = urb->context; + struct xillyusb_endpoint *ep = xb->ep; + unsigned long flags; + + if (urb->status && + (!(urb->status == -ENOENT || + urb->status == -ECONNRESET || + urb->status == -ESHUTDOWN))) + report_io_error(ep->xdev, -EIO); + + spin_lock_irqsave(&ep->buffers_lock, flags); + list_add_tail(&xb->entry, &ep->buffers); + ep->outstanding_urbs--; + spin_unlock_irqrestore(&ep->buffers_lock, flags); + + if (!ep->shutting_down) + queue_work(ep->xdev->workq, &ep->workitem); +} + +static void try_queue_bulk_in(struct xillyusb_endpoint *ep) +{ + struct xillyusb_dev *xdev = ep->xdev; + struct xillybuffer *xb; + struct urb *urb; + + int rc; + unsigned long flags; + unsigned int bufsize = ep->buffer_size; + + mutex_lock(&ep->ep_mutex); + + if (ep->shutting_down || xdev->error) + goto done; + + while (1) { + spin_lock_irqsave(&ep->buffers_lock, flags); + + if (list_empty(&ep->buffers)) { + spin_unlock_irqrestore(&ep->buffers_lock, flags); + goto done; + } + + xb = list_first_entry(&ep->buffers, struct xillybuffer, entry); + list_del(&xb->entry); + ep->outstanding_urbs++; + + spin_unlock_irqrestore(&ep->buffers_lock, flags); + + urb = usb_alloc_urb(0, GFP_KERNEL); + if (!urb) { + report_io_error(xdev, -ENOMEM); + goto relist; + } + + usb_fill_bulk_urb(urb, xdev->udev, + usb_rcvbulkpipe(xdev->udev, ep->ep_num), + xb->buf, bufsize, bulk_in_completer, xb); + + usb_anchor_urb(urb, &ep->anchor); + + rc = usb_submit_urb(urb, GFP_KERNEL); + + if (rc) { + report_io_error(xdev, (rc == -ENOMEM) ? -ENOMEM : + -EIO); + goto unanchor; + } + + usb_free_urb(urb); /* This just decrements reference count */ + } + +unanchor: + usb_unanchor_urb(urb); + usb_free_urb(urb); + +relist: + spin_lock_irqsave(&ep->buffers_lock, flags); + list_add_tail(&xb->entry, &ep->buffers); + ep->outstanding_urbs--; + spin_unlock_irqrestore(&ep->buffers_lock, flags); + +done: + mutex_unlock(&ep->ep_mutex); +} + +static void try_queue_bulk_out(struct xillyusb_endpoint *ep) +{ + struct xillyfifo *fifo = &ep->fifo; + struct xillyusb_dev *xdev = ep->xdev; + struct xillybuffer *xb; + struct urb *urb; + + int rc; + unsigned int fill; + unsigned long flags; + bool do_wake = false; + + mutex_lock(&ep->ep_mutex); + + if (ep->shutting_down || xdev->error) + goto done; + + fill = READ_ONCE(fifo->fill) & ep->fill_mask; + + while (1) { + int count; + unsigned int max_read; + + spin_lock_irqsave(&ep->buffers_lock, flags); + + /* + * Race conditions might have the FIFO filled while the + * endpoint is marked as drained here. That doesn't matter, + * because the sole purpose of @drained is to ensure that + * certain data has been sent on the USB channel before + * shutting it down. Hence knowing that the FIFO appears + * to be empty with no outstanding URBs at some moment + * is good enough. + */ + + if (!fill) { + ep->drained = !ep->outstanding_urbs; + if (ep->drained && ep->wake_on_drain) + do_wake = true; + + spin_unlock_irqrestore(&ep->buffers_lock, flags); + goto done; + } + + ep->drained = false; + + if ((fill < ep->buffer_size && ep->outstanding_urbs) || + list_empty(&ep->buffers)) { + spin_unlock_irqrestore(&ep->buffers_lock, flags); + goto done; + } + + xb = list_first_entry(&ep->buffers, struct xillybuffer, entry); + list_del(&xb->entry); + ep->outstanding_urbs++; + + spin_unlock_irqrestore(&ep->buffers_lock, flags); + + max_read = min(fill, ep->buffer_size); + + count = fifo_read(&ep->fifo, xb->buf, max_read, xilly_memcpy); + + /* + * xilly_memcpy always returns 0 => fifo_read can't fail => + * count > 0 + */ + + urb = usb_alloc_urb(0, GFP_KERNEL); + if (!urb) { + report_io_error(xdev, -ENOMEM); + goto relist; + } + + usb_fill_bulk_urb(urb, xdev->udev, + usb_sndbulkpipe(xdev->udev, ep->ep_num), + xb->buf, count, bulk_out_completer, xb); + + usb_anchor_urb(urb, &ep->anchor); + + rc = usb_submit_urb(urb, GFP_KERNEL); + + if (rc) { + report_io_error(xdev, (rc == -ENOMEM) ? -ENOMEM : + -EIO); + goto unanchor; + } + + usb_free_urb(urb); /* This just decrements reference count */ + + fill -= count; + do_wake = true; + } + +unanchor: + usb_unanchor_urb(urb); + usb_free_urb(urb); + +relist: + spin_lock_irqsave(&ep->buffers_lock, flags); + list_add_tail(&xb->entry, &ep->buffers); + ep->outstanding_urbs--; + spin_unlock_irqrestore(&ep->buffers_lock, flags); + +done: + mutex_unlock(&ep->ep_mutex); + + if (do_wake) + wake_up_interruptible(&fifo->waitq); +} + +static void bulk_out_work(struct work_struct *work) +{ + struct xillyusb_endpoint *ep = container_of(work, + struct xillyusb_endpoint, + workitem); + try_queue_bulk_out(ep); +} + +static int process_in_opcode(struct xillyusb_dev *xdev, + int opcode, + int chan_num) +{ + struct xillyusb_channel *chan; + struct device *dev = xdev->dev; + int chan_idx = chan_num >> 1; + + if (chan_idx >= xdev->num_channels) { + dev_err(dev, "Received illegal channel ID %d from FPGA\n", + chan_num); + return -EIO; + } + + chan = &xdev->channels[chan_idx]; + + switch (opcode) { + case OPCODE_EOF: + if (!chan->read_data_ok) { + dev_err(dev, "Received unexpected EOF for channel %d\n", + chan_num); + return -EIO; + } + + /* + * A write memory barrier ensures that the FIFO's fill level + * is visible before read_data_ok turns zero, so the data in + * the FIFO isn't missed by the consumer. + */ + smp_wmb(); + WRITE_ONCE(chan->read_data_ok, 0); + wake_up_interruptible(&chan->in_fifo->waitq); + break; + + case OPCODE_REACHED_CHECKPOINT: + chan->flushing = 0; + wake_up_interruptible(&chan->flushq); + break; + + case OPCODE_CANCELED_CHECKPOINT: + chan->canceled = 1; + wake_up_interruptible(&chan->flushq); + break; + + default: + dev_err(dev, "Received illegal opcode %d from FPGA\n", + opcode); + return -EIO; + } + + return 0; +} + +static int process_bulk_in(struct xillybuffer *xb) +{ + struct xillyusb_endpoint *ep = xb->ep; + struct xillyusb_dev *xdev = ep->xdev; + struct device *dev = xdev->dev; + int dws = xb->len >> 2; + __le32 *p = xb->buf; + u32 ctrlword; + struct xillyusb_channel *chan; + struct xillyfifo *fifo; + int chan_num = 0, opcode; + int chan_idx; + int bytes, count, dwconsume; + int in_bytes_left = 0; + int rc; + + if ((dws << 2) != xb->len) { + dev_err(dev, "Received BULK IN transfer with %d bytes, not a multiple of 4\n", + xb->len); + return -EIO; + } + + if (xdev->in_bytes_left) { + bytes = min(xdev->in_bytes_left, dws << 2); + in_bytes_left = xdev->in_bytes_left - bytes; + chan_num = xdev->leftover_chan_num; + goto resume_leftovers; + } + + while (dws) { + ctrlword = le32_to_cpu(*p++); + dws--; + + chan_num = ctrlword & 0xfff; + count = (ctrlword >> 12) & 0x3ff; + opcode = (ctrlword >> 24) & 0xf; + + if (opcode != OPCODE_DATA) { + unsigned int in_counter = xdev->in_counter++ & 0x3ff; + + if (count != in_counter) { + dev_err(dev, "Expected opcode counter %d, got %d\n", + in_counter, count); + return -EIO; + } + + rc = process_in_opcode(xdev, opcode, chan_num); + + if (rc) + return rc; + + continue; + } + + bytes = min(count + 1, dws << 2); + in_bytes_left = count + 1 - bytes; + +resume_leftovers: + chan_idx = chan_num >> 1; + + if (!(chan_num & 1) || chan_idx >= xdev->num_channels || + !xdev->channels[chan_idx].read_data_ok) { + dev_err(dev, "Received illegal channel ID %d from FPGA\n", + chan_num); + return -EIO; + } + chan = &xdev->channels[chan_idx]; + + fifo = chan->in_fifo; + + if (unlikely(!fifo)) + return -EIO; /* We got really unexpected data */ + + if (bytes != fifo_write(fifo, p, bytes, xilly_memcpy)) { + dev_err(dev, "Misbehaving FPGA overflowed an upstream FIFO!\n"); + return -EIO; + } + + wake_up_interruptible(&fifo->waitq); + + dwconsume = (bytes + 3) >> 2; + dws -= dwconsume; + p += dwconsume; + } + + xdev->in_bytes_left = in_bytes_left; + xdev->leftover_chan_num = chan_num; + return 0; +} + +static void bulk_in_work(struct work_struct *work) +{ + struct xillyusb_endpoint *ep = + container_of(work, struct xillyusb_endpoint, workitem); + struct xillyusb_dev *xdev = ep->xdev; + unsigned long flags; + struct xillybuffer *xb; + bool consumed = false; + int rc = 0; + + mutex_lock(&xdev->process_in_mutex); + + spin_lock_irqsave(&ep->buffers_lock, flags); + + while (1) { + if (rc || list_empty(&ep->filled_buffers)) { + spin_unlock_irqrestore(&ep->buffers_lock, flags); + mutex_unlock(&xdev->process_in_mutex); + + if (rc) + report_io_error(xdev, rc); + else if (consumed) + try_queue_bulk_in(ep); + + return; + } + + xb = list_first_entry(&ep->filled_buffers, struct xillybuffer, + entry); + list_del(&xb->entry); + + spin_unlock_irqrestore(&ep->buffers_lock, flags); + + consumed = true; + + if (!xdev->error) + rc = process_bulk_in(xb); + + spin_lock_irqsave(&ep->buffers_lock, flags); + list_add_tail(&xb->entry, &ep->buffers); + ep->outstanding_urbs--; + } +} + +static int xillyusb_send_opcode(struct xillyusb_dev *xdev, + int chan_num, char opcode, u32 data) +{ + struct xillyusb_endpoint *ep = xdev->msg_ep; + struct xillyfifo *fifo = &ep->fifo; + __le32 msg[2]; + + int rc = 0; + + msg[0] = cpu_to_le32((chan_num & 0xfff) | + ((opcode & 0xf) << 24)); + msg[1] = cpu_to_le32(data); + + mutex_lock(&xdev->msg_mutex); + + /* + * The wait queue is woken with the interruptible variant, so the + * wait function matches, however returning because of an interrupt + * will mess things up considerably, in particular when the caller is + * the release method. And the xdev->error part prevents being stuck + * forever in the event of a bizarre hardware bug: Pull the USB plug. + */ + + while (wait_event_interruptible(fifo->waitq, + fifo->fill <= (fifo->size - 8) || + xdev->error)) + ; /* Empty loop */ + + if (xdev->error) { + rc = xdev->error; + goto unlock_done; + } + + fifo_write(fifo, (void *)msg, 8, xilly_memcpy); + + try_queue_bulk_out(ep); + +unlock_done: + mutex_unlock(&xdev->msg_mutex); + + return rc; +} + +/* + * Note that flush_downstream() merely waits for the data to arrive to + * the application logic at the FPGA -- unlike PCIe Xillybus' counterpart, + * it does nothing to make it happen (and neither is it necessary). + * + * This function is not reentrant for the same @chan, but this is covered + * by the fact that for any given @chan, it's called either by the open, + * write, llseek and flush fops methods, which can't run in parallel (and the + * write + flush and llseek method handlers are protected with out_mutex). + * + * chan->flushed is there to avoid multiple flushes at the same position, + * in particular as a result of programs that close the file descriptor + * e.g. after a dup2() for redirection. + */ + +static int flush_downstream(struct xillyusb_channel *chan, + long timeout, + bool interruptible) +{ + struct xillyusb_dev *xdev = chan->xdev; + int chan_num = chan->chan_idx << 1; + long deadline, left_to_sleep; + int rc; + + if (chan->flushed) + return 0; + + deadline = jiffies + 1 + timeout; + + if (chan->flushing) { + long cancel_deadline = jiffies + 1 + XILLY_RESPONSE_TIMEOUT; + + chan->canceled = 0; + rc = xillyusb_send_opcode(xdev, chan_num, + OPCODE_CANCEL_CHECKPOINT, 0); + + if (rc) + return rc; /* Only real error, never -EINTR */ + + /* Ignoring interrupts. Cancellation must be handled */ + while (!chan->canceled) { + left_to_sleep = cancel_deadline - ((long)jiffies); + + if (left_to_sleep <= 0) { + report_io_error(xdev, -EIO); + return -EIO; + } + + rc = wait_event_interruptible_timeout(chan->flushq, + chan->canceled || + xdev->error, + left_to_sleep); + + if (xdev->error) + return xdev->error; + } + } + + chan->flushing = 1; + + /* + * The checkpoint is given in terms of data elements, not bytes. As + * a result, if less than an element's worth of data is stored in the + * FIFO, it's not flushed, including the flush before closing, which + * means that such data is lost. This is consistent with PCIe Xillybus. + */ + + rc = xillyusb_send_opcode(xdev, chan_num, + OPCODE_SET_CHECKPOINT, + chan->out_bytes >> + chan->out_log2_element_size); + + if (rc) + return rc; /* Only real error, never -EINTR */ + + if (!timeout) { + while (chan->flushing) { + rc = wait_event_interruptible(chan->flushq, + !chan->flushing || + xdev->error); + if (xdev->error) + return xdev->error; + + if (interruptible && rc) + return -EINTR; + } + + goto done; + } + + while (chan->flushing) { + left_to_sleep = deadline - ((long)jiffies); + + if (left_to_sleep <= 0) + return -ETIMEDOUT; + + rc = wait_event_interruptible_timeout(chan->flushq, + !chan->flushing || + xdev->error, + left_to_sleep); + + if (xdev->error) + return xdev->error; + + if (interruptible && rc < 0) + return -EINTR; + } + +done: + chan->flushed = 1; + return 0; +} + +/* request_read_anything(): Ask the FPGA for any little amount of data */ +static int request_read_anything(struct xillyusb_channel *chan, + char opcode) +{ + struct xillyusb_dev *xdev = chan->xdev; + unsigned int sh = chan->in_log2_element_size; + int chan_num = (chan->chan_idx << 1) | 1; + u32 mercy = chan->in_consumed_bytes + (2 << sh) - 1; + + return xillyusb_send_opcode(xdev, chan_num, opcode, mercy >> sh); +} + +static int xillyusb_open(struct inode *inode, struct file *filp) +{ + struct xillyusb_dev *xdev; + struct xillyusb_channel *chan; + struct xillyfifo *in_fifo = NULL; + struct xillyusb_endpoint *out_ep = NULL; + int rc; + int index; + + mutex_lock(&kref_mutex); + + rc = xillybus_find_inode(inode, (void **)&xdev, &index); + if (rc) { + mutex_unlock(&kref_mutex); + return rc; + } + + kref_get(&xdev->kref); + mutex_unlock(&kref_mutex); + + chan = &xdev->channels[index]; + filp->private_data = chan; + + mutex_lock(&chan->lock); + + rc = -ENODEV; + + if (xdev->error) + goto unmutex_fail; + + if (((filp->f_mode & FMODE_READ) && !chan->readable) || + ((filp->f_mode & FMODE_WRITE) && !chan->writable)) + goto unmutex_fail; + + if ((filp->f_flags & O_NONBLOCK) && (filp->f_mode & FMODE_READ) && + chan->in_synchronous) { + dev_err(xdev->dev, + "open() failed: O_NONBLOCK not allowed for read on this device\n"); + goto unmutex_fail; + } + + if ((filp->f_flags & O_NONBLOCK) && (filp->f_mode & FMODE_WRITE) && + chan->out_synchronous) { + dev_err(xdev->dev, + "open() failed: O_NONBLOCK not allowed for write on this device\n"); + goto unmutex_fail; + } + + rc = -EBUSY; + + if (((filp->f_mode & FMODE_READ) && chan->open_for_read) || + ((filp->f_mode & FMODE_WRITE) && chan->open_for_write)) + goto unmutex_fail; + + if (filp->f_mode & FMODE_READ) + chan->open_for_read = 1; + + if (filp->f_mode & FMODE_WRITE) + chan->open_for_write = 1; + + mutex_unlock(&chan->lock); + + if (filp->f_mode & FMODE_WRITE) { + out_ep = endpoint_alloc(xdev, + (chan->chan_idx + 2) | USB_DIR_OUT, + bulk_out_work, BUF_SIZE_ORDER, BUFNUM); + + if (!out_ep) { + rc = -ENOMEM; + goto unopen; + } + + rc = fifo_init(&out_ep->fifo, chan->out_log2_fifo_size); + + if (rc) + goto late_unopen; + + out_ep->fill_mask = -(1 << chan->out_log2_element_size); + chan->out_bytes = 0; + chan->flushed = 0; + + /* + * Sending a flush request to a previously closed stream + * effectively opens it, and also waits until the command is + * confirmed by the FPGA. The latter is necessary because the + * data is sent through a separate BULK OUT endpoint, and the + * xHCI controller is free to reorder transmissions. + * + * This can't go wrong unless there's a serious hardware error + * (or the computer is stuck for 500 ms?) + */ + rc = flush_downstream(chan, XILLY_RESPONSE_TIMEOUT, false); + + if (rc == -ETIMEDOUT) { + rc = -EIO; + report_io_error(xdev, rc); + } + + if (rc) + goto late_unopen; + } + + if (filp->f_mode & FMODE_READ) { + in_fifo = kzalloc(sizeof(*in_fifo), GFP_KERNEL); + + if (!in_fifo) { + rc = -ENOMEM; + goto late_unopen; + } + + rc = fifo_init(in_fifo, chan->in_log2_fifo_size); + + if (rc) { + kfree(in_fifo); + goto late_unopen; + } + } + + mutex_lock(&chan->lock); + if (in_fifo) { + chan->in_fifo = in_fifo; + chan->read_data_ok = 1; + } + if (out_ep) + chan->out_ep = out_ep; + mutex_unlock(&chan->lock); + + if (in_fifo) { + u32 in_checkpoint = 0; + + if (!chan->in_synchronous) + in_checkpoint = in_fifo->size >> + chan->in_log2_element_size; + + chan->in_consumed_bytes = 0; + chan->poll_used = 0; + chan->in_current_checkpoint = in_checkpoint; + rc = xillyusb_send_opcode(xdev, (chan->chan_idx << 1) | 1, + OPCODE_SET_CHECKPOINT, + in_checkpoint); + + if (rc) /* Failure guarantees that opcode wasn't sent */ + goto unfifo; + + /* + * In non-blocking mode, request the FPGA to send any data it + * has right away. Otherwise, the first read() will always + * return -EAGAIN, which is OK strictly speaking, but ugly. + * Checking and unrolling if this fails isn't worth the + * effort -- the error is propagated to the first read() + * anyhow. + */ + if (filp->f_flags & O_NONBLOCK) + request_read_anything(chan, OPCODE_SET_PUSH); + } + + return 0; + +unfifo: + chan->read_data_ok = 0; + safely_assign_in_fifo(chan, NULL); + fifo_mem_release(in_fifo); + kfree(in_fifo); + + if (out_ep) { + mutex_lock(&chan->lock); + chan->out_ep = NULL; + mutex_unlock(&chan->lock); + } + +late_unopen: + if (out_ep) + endpoint_dealloc(out_ep); + +unopen: + mutex_lock(&chan->lock); + + if (filp->f_mode & FMODE_READ) + chan->open_for_read = 0; + + if (filp->f_mode & FMODE_WRITE) + chan->open_for_write = 0; + + mutex_unlock(&chan->lock); + + kref_put(&xdev->kref, cleanup_dev); + + return rc; + +unmutex_fail: + kref_put(&xdev->kref, cleanup_dev); + mutex_unlock(&chan->lock); + return rc; +} + +static ssize_t xillyusb_read(struct file *filp, char __user *userbuf, + size_t count, loff_t *f_pos) +{ + struct xillyusb_channel *chan = filp->private_data; + struct xillyusb_dev *xdev = chan->xdev; + struct xillyfifo *fifo = chan->in_fifo; + int chan_num = (chan->chan_idx << 1) | 1; + + long deadline, left_to_sleep; + int bytes_done = 0; + bool sent_set_push = false; + int rc; + + deadline = jiffies + 1 + XILLY_RX_TIMEOUT; + + rc = mutex_lock_interruptible(&chan->in_mutex); + + if (rc) + return rc; + + while (1) { + u32 fifo_checkpoint_bytes, complete_checkpoint_bytes; + u32 complete_checkpoint, fifo_checkpoint; + u32 checkpoint; + s32 diff, leap; + unsigned int sh = chan->in_log2_element_size; + bool checkpoint_for_complete; + + rc = fifo_read(fifo, (__force void *)userbuf + bytes_done, + count - bytes_done, xilly_copy_to_user); + + if (rc < 0) + break; + + bytes_done += rc; + chan->in_consumed_bytes += rc; + + left_to_sleep = deadline - ((long)jiffies); + + /* + * Some 32-bit arithmetic that may wrap. Note that + * complete_checkpoint is rounded up to the closest element + * boundary, because the read() can't be completed otherwise. + * fifo_checkpoint_bytes is rounded down, because it protects + * in_fifo from overflowing. + */ + + fifo_checkpoint_bytes = chan->in_consumed_bytes + fifo->size; + complete_checkpoint_bytes = + chan->in_consumed_bytes + count - bytes_done; + + fifo_checkpoint = fifo_checkpoint_bytes >> sh; + complete_checkpoint = + (complete_checkpoint_bytes + (1 << sh) - 1) >> sh; + + diff = (fifo_checkpoint - complete_checkpoint) << sh; + + if (chan->in_synchronous && diff >= 0) { + checkpoint = complete_checkpoint; + checkpoint_for_complete = true; + } else { + checkpoint = fifo_checkpoint; + checkpoint_for_complete = false; + } + + leap = (checkpoint - chan->in_current_checkpoint) << sh; + + /* + * To prevent flooding of OPCODE_SET_CHECKPOINT commands as + * data is consumed, it's issued only if it moves the + * checkpoint by at least an 8th of the FIFO's size, or if + * it's necessary to complete the number of bytes requested by + * the read() call. + * + * chan->read_data_ok is checked to spare an unnecessary + * submission after receiving EOF, however it's harmless if + * such slips away. + */ + + if (chan->read_data_ok && + (leap > (fifo->size >> 3) || + (checkpoint_for_complete && leap > 0))) { + chan->in_current_checkpoint = checkpoint; + rc = xillyusb_send_opcode(xdev, chan_num, + OPCODE_SET_CHECKPOINT, + checkpoint); + + if (rc) + break; + } + + if (bytes_done == count || + (left_to_sleep <= 0 && bytes_done)) + break; + + /* + * Reaching here means that the FIFO was empty when + * fifo_read() returned, but not necessarily right now. Error + * and EOF are checked and reported only now, so that no data + * that managed its way to the FIFO is lost. + */ + + if (!READ_ONCE(chan->read_data_ok)) { /* FPGA has sent EOF */ + /* Has data slipped into the FIFO since fifo_read()? */ + smp_rmb(); + if (READ_ONCE(fifo->fill)) + continue; + + rc = 0; + break; + } + + if (xdev->error) { + rc = xdev->error; + break; + } + + if (filp->f_flags & O_NONBLOCK) { + rc = -EAGAIN; + break; + } + + if (!sent_set_push) { + rc = xillyusb_send_opcode(xdev, chan_num, + OPCODE_SET_PUSH, + complete_checkpoint); + + if (rc) + break; + + sent_set_push = true; + } + + if (left_to_sleep > 0) { + /* + * Note that when xdev->error is set (e.g. when the + * device is unplugged), read_data_ok turns zero and + * fifo->waitq is awaken. + * Therefore no special attention to xdev->error. + */ + + rc = wait_event_interruptible_timeout + (fifo->waitq, + fifo->fill || !chan->read_data_ok, + left_to_sleep); + } else { /* bytes_done == 0 */ + /* Tell FPGA to send anything it has */ + rc = request_read_anything(chan, OPCODE_UPDATE_PUSH); + + if (rc) + break; + + rc = wait_event_interruptible + (fifo->waitq, + fifo->fill || !chan->read_data_ok); + } + + if (rc < 0) { + rc = -EINTR; + break; + } + } + + if (((filp->f_flags & O_NONBLOCK) || chan->poll_used) && + !READ_ONCE(fifo->fill)) + request_read_anything(chan, OPCODE_SET_PUSH); + + mutex_unlock(&chan->in_mutex); + + if (bytes_done) + return bytes_done; + + return rc; +} + +static int xillyusb_flush(struct file *filp, fl_owner_t id) +{ + struct xillyusb_channel *chan = filp->private_data; + int rc; + + if (!(filp->f_mode & FMODE_WRITE)) + return 0; + + rc = mutex_lock_interruptible(&chan->out_mutex); + + if (rc) + return rc; + + /* + * One second's timeout on flushing. Interrupts are ignored, because if + * the user pressed CTRL-C, that interrupt will still be in flight by + * the time we reach here, and the opportunity to flush is lost. + */ + rc = flush_downstream(chan, HZ, false); + + mutex_unlock(&chan->out_mutex); + + if (rc == -ETIMEDOUT) { + /* The things you do to use dev_warn() and not pr_warn() */ + struct xillyusb_dev *xdev = chan->xdev; + + mutex_lock(&chan->lock); + if (!xdev->error) + dev_warn(xdev->dev, + "Timed out while flushing. Output data may be lost.\n"); + mutex_unlock(&chan->lock); + } + + return rc; +} + +static ssize_t xillyusb_write(struct file *filp, const char __user *userbuf, + size_t count, loff_t *f_pos) +{ + struct xillyusb_channel *chan = filp->private_data; + struct xillyusb_dev *xdev = chan->xdev; + struct xillyfifo *fifo = &chan->out_ep->fifo; + int rc; + + rc = mutex_lock_interruptible(&chan->out_mutex); + + if (rc) + return rc; + + while (1) { + if (xdev->error) { + rc = xdev->error; + break; + } + + if (count == 0) + break; + + rc = fifo_write(fifo, (__force void *)userbuf, count, + xilly_copy_from_user); + + if (rc != 0) + break; + + if (filp->f_flags & O_NONBLOCK) { + rc = -EAGAIN; + break; + } + + if (wait_event_interruptible + (fifo->waitq, + fifo->fill != fifo->size || xdev->error)) { + rc = -EINTR; + break; + } + } + + if (rc < 0) + goto done; + + chan->out_bytes += rc; + + if (rc) { + try_queue_bulk_out(chan->out_ep); + chan->flushed = 0; + } + + if (chan->out_synchronous) { + int flush_rc = flush_downstream(chan, 0, true); + + if (flush_rc && !rc) + rc = flush_rc; + } + +done: + mutex_unlock(&chan->out_mutex); + + return rc; +} + +static int xillyusb_release(struct inode *inode, struct file *filp) +{ + struct xillyusb_channel *chan = filp->private_data; + struct xillyusb_dev *xdev = chan->xdev; + int rc_read = 0, rc_write = 0; + + if (filp->f_mode & FMODE_READ) { + struct xillyfifo *in_fifo = chan->in_fifo; + + rc_read = xillyusb_send_opcode(xdev, (chan->chan_idx << 1) | 1, + OPCODE_CLOSE, 0); + /* + * If rc_read is nonzero, xdev->error indicates a global + * device error. The error is reported later, so that + * resources are freed. + * + * Looping on wait_event_interruptible() kinda breaks the idea + * of being interruptible, and this should have been + * wait_event(). Only it's being waken with + * wake_up_interruptible() for the sake of other uses. If + * there's a global device error, chan->read_data_ok is + * deasserted and the wait queue is awaken, so this is covered. + */ + + while (wait_event_interruptible(in_fifo->waitq, + !chan->read_data_ok)) + ; /* Empty loop */ + + safely_assign_in_fifo(chan, NULL); + fifo_mem_release(in_fifo); + kfree(in_fifo); + + mutex_lock(&chan->lock); + chan->open_for_read = 0; + mutex_unlock(&chan->lock); + } + + if (filp->f_mode & FMODE_WRITE) { + struct xillyusb_endpoint *ep = chan->out_ep; + /* + * chan->flushing isn't zeroed. If the pre-release flush timed + * out, a cancel request will be sent before the next + * OPCODE_SET_CHECKPOINT (i.e. when the file is opened again). + * This is despite that the FPGA forgets about the checkpoint + * request as the file closes. Still, in an exceptional race + * condition, the FPGA could send an OPCODE_REACHED_CHECKPOINT + * just before closing that would reach the host after the + * file has re-opened. + */ + + mutex_lock(&chan->lock); + chan->out_ep = NULL; + mutex_unlock(&chan->lock); + + endpoint_quiesce(ep); + endpoint_dealloc(ep); + + /* See comments on rc_read above */ + rc_write = xillyusb_send_opcode(xdev, chan->chan_idx << 1, + OPCODE_CLOSE, 0); + + mutex_lock(&chan->lock); + chan->open_for_write = 0; + mutex_unlock(&chan->lock); + } + + kref_put(&xdev->kref, cleanup_dev); + + return rc_read ? rc_read : rc_write; +} + +/* + * Xillybus' API allows device nodes to be seekable, giving the user + * application access to a RAM array on the FPGA (or logic emulating it). + */ + +static loff_t xillyusb_llseek(struct file *filp, loff_t offset, int whence) +{ + struct xillyusb_channel *chan = filp->private_data; + struct xillyusb_dev *xdev = chan->xdev; + loff_t pos = filp->f_pos; + int rc = 0; + unsigned int log2_element_size = chan->readable ? + chan->in_log2_element_size : chan->out_log2_element_size; + + /* + * Take both mutexes not allowing interrupts, since it seems like + * common applications don't expect an -EINTR here. Besides, multiple + * access to a single file descriptor on seekable devices is a mess + * anyhow. + */ + + mutex_lock(&chan->out_mutex); + mutex_lock(&chan->in_mutex); + + switch (whence) { + case SEEK_SET: + pos = offset; + break; + case SEEK_CUR: + pos += offset; + break; + case SEEK_END: + pos = offset; /* Going to the end => to the beginning */ + break; + default: + rc = -EINVAL; + goto end; + } + + /* In any case, we must finish on an element boundary */ + if (pos & ((1 << log2_element_size) - 1)) { + rc = -EINVAL; + goto end; + } + + rc = xillyusb_send_opcode(xdev, chan->chan_idx << 1, + OPCODE_SET_ADDR, + pos >> log2_element_size); + + if (rc) + goto end; + + if (chan->writable) { + chan->flushed = 0; + rc = flush_downstream(chan, HZ, false); + } + +end: + mutex_unlock(&chan->out_mutex); + mutex_unlock(&chan->in_mutex); + + if (rc) /* Return error after releasing mutexes */ + return rc; + + filp->f_pos = pos; + + return pos; +} + +static __poll_t xillyusb_poll(struct file *filp, poll_table *wait) +{ + struct xillyusb_channel *chan = filp->private_data; + __poll_t mask = 0; + + if (chan->in_fifo) + poll_wait(filp, &chan->in_fifo->waitq, wait); + + if (chan->out_ep) + poll_wait(filp, &chan->out_ep->fifo.waitq, wait); + + /* + * If this is the first time poll() is called, and the file is + * readable, set the relevant flag. Also tell the FPGA to send all it + * has, to kickstart the mechanism that ensures there's always some + * data in in_fifo unless the stream is dry end-to-end. Note that the + * first poll() may not return a EPOLLIN, even if there's data on the + * FPGA. Rather, the data will arrive soon, and trigger the relevant + * wait queue. + */ + + if (!chan->poll_used && chan->in_fifo) { + chan->poll_used = 1; + request_read_anything(chan, OPCODE_SET_PUSH); + } + + /* + * poll() won't play ball regarding read() channels which + * are synchronous. Allowing that will create situations where data has + * been delivered at the FPGA, and users expecting select() to wake up, + * which it may not. So make it never work. + */ + + if (chan->in_fifo && !chan->in_synchronous && + (READ_ONCE(chan->in_fifo->fill) || !chan->read_data_ok)) + mask |= EPOLLIN | EPOLLRDNORM; + + if (chan->out_ep && + (READ_ONCE(chan->out_ep->fifo.fill) != chan->out_ep->fifo.size)) + mask |= EPOLLOUT | EPOLLWRNORM; + + if (chan->xdev->error) + mask |= EPOLLERR; + + return mask; +} + +static const struct file_operations xillyusb_fops = { + .owner = THIS_MODULE, + .read = xillyusb_read, + .write = xillyusb_write, + .open = xillyusb_open, + .flush = xillyusb_flush, + .release = xillyusb_release, + .llseek = xillyusb_llseek, + .poll = xillyusb_poll, +}; + +static int xillyusb_setup_base_eps(struct xillyusb_dev *xdev) +{ + xdev->msg_ep = endpoint_alloc(xdev, MSG_EP_NUM | USB_DIR_OUT, + bulk_out_work, 1, 2); + if (!xdev->msg_ep) + return -ENOMEM; + + if (fifo_init(&xdev->msg_ep->fifo, 13)) /* 8 kiB */ + goto dealloc; + + xdev->msg_ep->fill_mask = -8; /* 8 bytes granularity */ + + xdev->in_ep = endpoint_alloc(xdev, IN_EP_NUM | USB_DIR_IN, + bulk_in_work, BUF_SIZE_ORDER, BUFNUM); + if (!xdev->in_ep) + goto dealloc; + + try_queue_bulk_in(xdev->in_ep); + + return 0; + +dealloc: + endpoint_dealloc(xdev->msg_ep); /* Also frees FIFO mem if allocated */ + xdev->msg_ep = NULL; + return -ENOMEM; +} + +static int setup_channels(struct xillyusb_dev *xdev, + __le16 *chandesc, + int num_channels) +{ + struct xillyusb_channel *chan; + int i; + + chan = kcalloc(num_channels, sizeof(*chan), GFP_KERNEL); + if (!chan) + return -ENOMEM; + + xdev->channels = chan; + + for (i = 0; i < num_channels; i++, chan++) { + unsigned int in_desc = le16_to_cpu(*chandesc++); + unsigned int out_desc = le16_to_cpu(*chandesc++); + + chan->xdev = xdev; + mutex_init(&chan->in_mutex); + mutex_init(&chan->out_mutex); + mutex_init(&chan->lock); + init_waitqueue_head(&chan->flushq); + + chan->chan_idx = i; + + if (in_desc & 0x80) { /* Entry is valid */ + chan->readable = 1; + chan->in_synchronous = !!(in_desc & 0x40); + chan->in_seekable = !!(in_desc & 0x20); + chan->in_log2_element_size = in_desc & 0x0f; + chan->in_log2_fifo_size = ((in_desc >> 8) & 0x1f) + 16; + } + + /* + * A downstream channel should never exist above index 13, + * as it would request a nonexistent BULK endpoint > 15. + * In the peculiar case that it does, it's ignored silently. + */ + + if ((out_desc & 0x80) && i < 14) { /* Entry is valid */ + chan->writable = 1; + chan->out_synchronous = !!(out_desc & 0x40); + chan->out_seekable = !!(out_desc & 0x20); + chan->out_log2_element_size = out_desc & 0x0f; + chan->out_log2_fifo_size = + ((out_desc >> 8) & 0x1f) + 16; + } + } + + return 0; +} + +static int xillyusb_discovery(struct usb_interface *interface) +{ + int rc; + struct xillyusb_dev *xdev = usb_get_intfdata(interface); + __le16 bogus_chandesc[2]; + struct xillyfifo idt_fifo; + struct xillyusb_channel *chan; + unsigned int idt_len, names_offset; + unsigned char *idt; + int num_channels; + + rc = xillyusb_send_opcode(xdev, ~0, OPCODE_QUIESCE, 0); + + if (rc) { + dev_err(&interface->dev, "Failed to send quiesce request. Aborting.\n"); + return rc; + } + + /* Phase I: Set up one fake upstream channel and obtain IDT */ + + /* Set up a fake IDT with one async IN stream */ + bogus_chandesc[0] = cpu_to_le16(0x80); + bogus_chandesc[1] = cpu_to_le16(0); + + rc = setup_channels(xdev, bogus_chandesc, 1); + + if (rc) + return rc; + + rc = fifo_init(&idt_fifo, LOG2_IDT_FIFO_SIZE); + + if (rc) + return rc; + + chan = xdev->channels; + + chan->in_fifo = &idt_fifo; + chan->read_data_ok = 1; + + xdev->num_channels = 1; + + rc = xillyusb_send_opcode(xdev, ~0, OPCODE_REQ_IDT, 0); + + if (rc) { + dev_err(&interface->dev, "Failed to send IDT request. Aborting.\n"); + goto unfifo; + } + + rc = wait_event_interruptible_timeout(idt_fifo.waitq, + !chan->read_data_ok, + XILLY_RESPONSE_TIMEOUT); + + if (xdev->error) { + rc = xdev->error; + goto unfifo; + } + + if (rc < 0) { + rc = -EINTR; /* Interrupt on probe method? Interesting. */ + goto unfifo; + } + + if (chan->read_data_ok) { + rc = -ETIMEDOUT; + dev_err(&interface->dev, "No response from FPGA. Aborting.\n"); + goto unfifo; + } + + idt_len = READ_ONCE(idt_fifo.fill); + idt = kmalloc(idt_len, GFP_KERNEL); + + if (!idt) { + rc = -ENOMEM; + goto unfifo; + } + + fifo_read(&idt_fifo, idt, idt_len, xilly_memcpy); + + if (crc32_le(~0, idt, idt_len) != 0) { + dev_err(&interface->dev, "IDT failed CRC check. Aborting.\n"); + rc = -ENODEV; + goto unidt; + } + + if (*idt > 0x90) { + dev_err(&interface->dev, "No support for IDT version 0x%02x. Maybe the xillyusb driver needs an upgrade. Aborting.\n", + (int)*idt); + rc = -ENODEV; + goto unidt; + } + + /* Phase II: Set up the streams as defined in IDT */ + + num_channels = le16_to_cpu(*((__le16 *)(idt + 1))); + names_offset = 3 + num_channels * 4; + idt_len -= 4; /* Exclude CRC */ + + if (idt_len < names_offset) { + dev_err(&interface->dev, "IDT too short. This is exceptionally weird, because its CRC is OK\n"); + rc = -ENODEV; + goto unidt; + } + + rc = setup_channels(xdev, (void *)idt + 3, num_channels); + + if (rc) + goto unidt; + + /* + * Except for wildly misbehaving hardware, or if it was disconnected + * just after responding with the IDT, there is no reason for any + * work item to be running now. To be sure that xdev->channels + * is updated on anything that might run in parallel, flush the + * workqueue, which rarely does anything. + */ + flush_workqueue(xdev->workq); + + xdev->num_channels = num_channels; + + fifo_mem_release(&idt_fifo); + kfree(chan); + + rc = xillybus_init_chrdev(&interface->dev, &xillyusb_fops, + THIS_MODULE, xdev, + idt + names_offset, + idt_len - names_offset, + num_channels, + xillyname, true); + + kfree(idt); + + return rc; + +unidt: + kfree(idt); + +unfifo: + safely_assign_in_fifo(chan, NULL); + fifo_mem_release(&idt_fifo); + + return rc; +} + +static int xillyusb_probe(struct usb_interface *interface, + const struct usb_device_id *id) +{ + struct xillyusb_dev *xdev; + int rc; + + xdev = kzalloc(sizeof(*xdev), GFP_KERNEL); + if (!xdev) + return -ENOMEM; + + kref_init(&xdev->kref); + mutex_init(&xdev->process_in_mutex); + mutex_init(&xdev->msg_mutex); + + xdev->udev = usb_get_dev(interface_to_usbdev(interface)); + xdev->dev = &interface->dev; + xdev->error = 0; + spin_lock_init(&xdev->error_lock); + xdev->in_counter = 0; + xdev->in_bytes_left = 0; + xdev->workq = alloc_workqueue(xillyname, WQ_HIGHPRI, 0); + + if (!xdev->workq) { + dev_err(&interface->dev, "Failed to allocate work queue\n"); + rc = -ENOMEM; + goto fail; + } + + INIT_WORK(&xdev->wakeup_workitem, wakeup_all); + + usb_set_intfdata(interface, xdev); + + rc = xillyusb_setup_base_eps(xdev); + if (rc) + goto fail; + + rc = xillyusb_discovery(interface); + if (rc) + goto latefail; + + return 0; + +latefail: + endpoint_quiesce(xdev->in_ep); + endpoint_quiesce(xdev->msg_ep); + +fail: + usb_set_intfdata(interface, NULL); + kref_put(&xdev->kref, cleanup_dev); + return rc; +} + +static void xillyusb_disconnect(struct usb_interface *interface) +{ + struct xillyusb_dev *xdev = usb_get_intfdata(interface); + struct xillyusb_endpoint *msg_ep = xdev->msg_ep; + struct xillyfifo *fifo = &msg_ep->fifo; + int rc; + int i; + + xillybus_cleanup_chrdev(xdev, &interface->dev); + + /* + * Try to send OPCODE_QUIESCE, which will fail silently if the device + * was disconnected, but makes sense on module unload. + */ + + msg_ep->wake_on_drain = true; + xillyusb_send_opcode(xdev, ~0, OPCODE_QUIESCE, 0); + + /* + * If the device has been disconnected, sending the opcode causes + * a global device error with xdev->error, if such error didn't + * occur earlier. Hence timing out means that the USB link is fine, + * but somehow the message wasn't sent. Should never happen. + */ + + rc = wait_event_interruptible_timeout(fifo->waitq, + msg_ep->drained || xdev->error, + XILLY_RESPONSE_TIMEOUT); + + if (!rc) + dev_err(&interface->dev, + "Weird timeout condition on sending quiesce request.\n"); + + report_io_error(xdev, -ENODEV); /* Discourage further activity */ + + /* + * This device driver is declared with soft_unbind set, or else + * sending OPCODE_QUIESCE above would always fail. The price is + * that the USB framework didn't kill outstanding URBs, so it has + * to be done explicitly before returning from this call. + */ + + for (i = 0; i < xdev->num_channels; i++) { + struct xillyusb_channel *chan = &xdev->channels[i]; + + /* + * Lock taken to prevent chan->out_ep from changing. It also + * ensures xillyusb_open() and xillyusb_flush() don't access + * xdev->dev after being nullified below. + */ + mutex_lock(&chan->lock); + if (chan->out_ep) + endpoint_quiesce(chan->out_ep); + mutex_unlock(&chan->lock); + } + + endpoint_quiesce(xdev->in_ep); + endpoint_quiesce(xdev->msg_ep); + + usb_set_intfdata(interface, NULL); + + xdev->dev = NULL; + + mutex_lock(&kref_mutex); + kref_put(&xdev->kref, cleanup_dev); + mutex_unlock(&kref_mutex); +} + +static struct usb_driver xillyusb_driver = { + .name = xillyname, + .id_table = xillyusb_table, + .probe = xillyusb_probe, + .disconnect = xillyusb_disconnect, + .soft_unbind = 1, +}; + +static int __init xillyusb_init(void) +{ + int rc = 0; + + if (LOG2_INITIAL_FIFO_BUF_SIZE > PAGE_SHIFT) + fifo_buf_order = LOG2_INITIAL_FIFO_BUF_SIZE - PAGE_SHIFT; + else + fifo_buf_order = 0; + + rc = usb_register(&xillyusb_driver); + + return rc; +} + +static void __exit xillyusb_exit(void) +{ + usb_deregister(&xillyusb_driver); +} + +module_init(xillyusb_init); +module_exit(xillyusb_exit); |