diff options
Diffstat (limited to 'drivers/char/xillybus/xillybus_core.c')
| -rw-r--r-- | drivers/char/xillybus/xillybus_core.c | 1991 |
1 files changed, 1991 insertions, 0 deletions
diff --git a/drivers/char/xillybus/xillybus_core.c b/drivers/char/xillybus/xillybus_core.c new file mode 100644 index 000000000..11b7c4749 --- /dev/null +++ b/drivers/char/xillybus/xillybus_core.c @@ -0,0 +1,1991 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/drivers/misc/xillybus_core.c + * + * Copyright 2011 Xillybus Ltd, http://xillybus.com + * + * Driver for the Xillybus FPGA/host framework. + * + * This driver interfaces with a special IP core in an FPGA, setting up + * a pipe between a hardware FIFO in the programmable logic and a device + * file in the host. The number of such pipes and their attributes are + * set up on the logic. This driver detects these automatically and + * creates the device files accordingly. + */ + +#include <linux/list.h> +#include <linux/device.h> +#include <linux/module.h> +#include <linux/io.h> +#include <linux/dma-mapping.h> +#include <linux/interrupt.h> +#include <linux/sched.h> +#include <linux/fs.h> +#include <linux/spinlock.h> +#include <linux/mutex.h> +#include <linux/crc32.h> +#include <linux/poll.h> +#include <linux/delay.h> +#include <linux/slab.h> +#include <linux/workqueue.h> +#include "xillybus.h" +#include "xillybus_class.h" + +MODULE_DESCRIPTION("Xillybus core functions"); +MODULE_AUTHOR("Eli Billauer, Xillybus Ltd."); +MODULE_ALIAS("xillybus_core"); +MODULE_LICENSE("GPL v2"); + +/* General timeout is 100 ms, rx timeout is 10 ms */ +#define XILLY_RX_TIMEOUT (10*HZ/1000) +#define XILLY_TIMEOUT (100*HZ/1000) + +#define fpga_msg_ctrl_reg 0x0008 +#define fpga_dma_control_reg 0x0020 +#define fpga_dma_bufno_reg 0x0024 +#define fpga_dma_bufaddr_lowaddr_reg 0x0028 +#define fpga_dma_bufaddr_highaddr_reg 0x002c +#define fpga_buf_ctrl_reg 0x0030 +#define fpga_buf_offset_reg 0x0034 +#define fpga_endian_reg 0x0040 + +#define XILLYMSG_OPCODE_RELEASEBUF 1 +#define XILLYMSG_OPCODE_QUIESCEACK 2 +#define XILLYMSG_OPCODE_FIFOEOF 3 +#define XILLYMSG_OPCODE_FATAL_ERROR 4 +#define XILLYMSG_OPCODE_NONEMPTY 5 + +static const char xillyname[] = "xillybus"; + +static struct workqueue_struct *xillybus_wq; + +/* + * Locking scheme: Mutexes protect invocations of character device methods. + * If both locks are taken, wr_mutex is taken first, rd_mutex second. + * + * wr_spinlock protects wr_*_buf_idx, wr_empty, wr_sleepy, wr_ready and the + * buffers' end_offset fields against changes made by IRQ handler (and in + * theory, other file request handlers, but the mutex handles that). Nothing + * else. + * They are held for short direct memory manipulations. Needless to say, + * no mutex locking is allowed when a spinlock is held. + * + * rd_spinlock does the same with rd_*_buf_idx, rd_empty and end_offset. + * + * register_mutex is endpoint-specific, and is held when non-atomic + * register operations are performed. wr_mutex and rd_mutex may be + * held when register_mutex is taken, but none of the spinlocks. Note that + * register_mutex doesn't protect against sporadic buf_ctrl_reg writes + * which are unrelated to buf_offset_reg, since they are harmless. + * + * Blocking on the wait queues is allowed with mutexes held, but not with + * spinlocks. + * + * Only interruptible blocking is allowed on mutexes and wait queues. + * + * All in all, the locking order goes (with skips allowed, of course): + * wr_mutex -> rd_mutex -> register_mutex -> wr_spinlock -> rd_spinlock + */ + +static void malformed_message(struct xilly_endpoint *endpoint, u32 *buf) +{ + int opcode; + int msg_channel, msg_bufno, msg_data, msg_dir; + + opcode = (buf[0] >> 24) & 0xff; + msg_dir = buf[0] & 1; + msg_channel = (buf[0] >> 1) & 0x7ff; + msg_bufno = (buf[0] >> 12) & 0x3ff; + msg_data = buf[1] & 0xfffffff; + + dev_warn(endpoint->dev, + "Malformed message (skipping): opcode=%d, channel=%03x, dir=%d, bufno=%03x, data=%07x\n", + opcode, msg_channel, msg_dir, msg_bufno, msg_data); +} + +/* + * xillybus_isr assumes the interrupt is allocated exclusively to it, + * which is the natural case MSI and several other hardware-oriented + * interrupts. Sharing is not allowed. + */ + +irqreturn_t xillybus_isr(int irq, void *data) +{ + struct xilly_endpoint *ep = data; + u32 *buf; + unsigned int buf_size; + int i; + int opcode; + unsigned int msg_channel, msg_bufno, msg_data, msg_dir; + struct xilly_channel *channel; + + buf = ep->msgbuf_addr; + buf_size = ep->msg_buf_size/sizeof(u32); + + dma_sync_single_for_cpu(ep->dev, ep->msgbuf_dma_addr, + ep->msg_buf_size, DMA_FROM_DEVICE); + + for (i = 0; i < buf_size; i += 2) { + if (((buf[i+1] >> 28) & 0xf) != ep->msg_counter) { + malformed_message(ep, &buf[i]); + dev_warn(ep->dev, + "Sending a NACK on counter %x (instead of %x) on entry %d\n", + ((buf[i+1] >> 28) & 0xf), + ep->msg_counter, + i/2); + + if (++ep->failed_messages > 10) { + dev_err(ep->dev, + "Lost sync with interrupt messages. Stopping.\n"); + } else { + dma_sync_single_for_device(ep->dev, + ep->msgbuf_dma_addr, + ep->msg_buf_size, + DMA_FROM_DEVICE); + + iowrite32(0x01, /* Message NACK */ + ep->registers + fpga_msg_ctrl_reg); + } + return IRQ_HANDLED; + } else if (buf[i] & (1 << 22)) /* Last message */ + break; + } + + if (i >= buf_size) { + dev_err(ep->dev, "Bad interrupt message. Stopping.\n"); + return IRQ_HANDLED; + } + + buf_size = i + 2; + + for (i = 0; i < buf_size; i += 2) { /* Scan through messages */ + opcode = (buf[i] >> 24) & 0xff; + + msg_dir = buf[i] & 1; + msg_channel = (buf[i] >> 1) & 0x7ff; + msg_bufno = (buf[i] >> 12) & 0x3ff; + msg_data = buf[i+1] & 0xfffffff; + + switch (opcode) { + case XILLYMSG_OPCODE_RELEASEBUF: + if ((msg_channel > ep->num_channels) || + (msg_channel == 0)) { + malformed_message(ep, &buf[i]); + break; + } + + channel = ep->channels[msg_channel]; + + if (msg_dir) { /* Write channel */ + if (msg_bufno >= channel->num_wr_buffers) { + malformed_message(ep, &buf[i]); + break; + } + spin_lock(&channel->wr_spinlock); + channel->wr_buffers[msg_bufno]->end_offset = + msg_data; + channel->wr_fpga_buf_idx = msg_bufno; + channel->wr_empty = 0; + channel->wr_sleepy = 0; + spin_unlock(&channel->wr_spinlock); + + wake_up_interruptible(&channel->wr_wait); + + } else { + /* Read channel */ + + if (msg_bufno >= channel->num_rd_buffers) { + malformed_message(ep, &buf[i]); + break; + } + + spin_lock(&channel->rd_spinlock); + channel->rd_fpga_buf_idx = msg_bufno; + channel->rd_full = 0; + spin_unlock(&channel->rd_spinlock); + + wake_up_interruptible(&channel->rd_wait); + if (!channel->rd_synchronous) + queue_delayed_work( + xillybus_wq, + &channel->rd_workitem, + XILLY_RX_TIMEOUT); + } + + break; + case XILLYMSG_OPCODE_NONEMPTY: + if ((msg_channel > ep->num_channels) || + (msg_channel == 0) || (!msg_dir) || + !ep->channels[msg_channel]->wr_supports_nonempty) { + malformed_message(ep, &buf[i]); + break; + } + + channel = ep->channels[msg_channel]; + + if (msg_bufno >= channel->num_wr_buffers) { + malformed_message(ep, &buf[i]); + break; + } + spin_lock(&channel->wr_spinlock); + if (msg_bufno == channel->wr_host_buf_idx) + channel->wr_ready = 1; + spin_unlock(&channel->wr_spinlock); + + wake_up_interruptible(&channel->wr_ready_wait); + + break; + case XILLYMSG_OPCODE_QUIESCEACK: + ep->idtlen = msg_data; + wake_up_interruptible(&ep->ep_wait); + + break; + case XILLYMSG_OPCODE_FIFOEOF: + if ((msg_channel > ep->num_channels) || + (msg_channel == 0) || (!msg_dir) || + !ep->channels[msg_channel]->num_wr_buffers) { + malformed_message(ep, &buf[i]); + break; + } + channel = ep->channels[msg_channel]; + spin_lock(&channel->wr_spinlock); + channel->wr_eof = msg_bufno; + channel->wr_sleepy = 0; + + channel->wr_hangup = channel->wr_empty && + (channel->wr_host_buf_idx == msg_bufno); + + spin_unlock(&channel->wr_spinlock); + + wake_up_interruptible(&channel->wr_wait); + + break; + case XILLYMSG_OPCODE_FATAL_ERROR: + ep->fatal_error = 1; + wake_up_interruptible(&ep->ep_wait); /* For select() */ + dev_err(ep->dev, + "FPGA reported a fatal error. This means that the low-level communication with the device has failed. This hardware problem is most likely unrelated to Xillybus (neither kernel module nor FPGA core), but reports are still welcome. All I/O is aborted.\n"); + break; + default: + malformed_message(ep, &buf[i]); + break; + } + } + + dma_sync_single_for_device(ep->dev, ep->msgbuf_dma_addr, + ep->msg_buf_size, DMA_FROM_DEVICE); + + ep->msg_counter = (ep->msg_counter + 1) & 0xf; + ep->failed_messages = 0; + iowrite32(0x03, ep->registers + fpga_msg_ctrl_reg); /* Message ACK */ + + return IRQ_HANDLED; +} +EXPORT_SYMBOL(xillybus_isr); + +/* + * A few trivial memory management functions. + * NOTE: These functions are used only on probe and remove, and therefore + * no locks are applied! + */ + +static void xillybus_autoflush(struct work_struct *work); + +struct xilly_alloc_state { + void *salami; + int left_of_salami; + int nbuffer; + enum dma_data_direction direction; + u32 regdirection; +}; + +static void xilly_unmap(void *ptr) +{ + struct xilly_mapping *data = ptr; + + dma_unmap_single(data->device, data->dma_addr, + data->size, data->direction); + + kfree(ptr); +} + +static int xilly_map_single(struct xilly_endpoint *ep, + void *ptr, + size_t size, + int direction, + dma_addr_t *ret_dma_handle + ) +{ + dma_addr_t addr; + struct xilly_mapping *this; + + this = kzalloc(sizeof(*this), GFP_KERNEL); + if (!this) + return -ENOMEM; + + addr = dma_map_single(ep->dev, ptr, size, direction); + + if (dma_mapping_error(ep->dev, addr)) { + kfree(this); + return -ENODEV; + } + + this->device = ep->dev; + this->dma_addr = addr; + this->size = size; + this->direction = direction; + + *ret_dma_handle = addr; + + return devm_add_action_or_reset(ep->dev, xilly_unmap, this); +} + +static int xilly_get_dma_buffers(struct xilly_endpoint *ep, + struct xilly_alloc_state *s, + struct xilly_buffer **buffers, + int bufnum, int bytebufsize) +{ + int i, rc; + dma_addr_t dma_addr; + struct device *dev = ep->dev; + struct xilly_buffer *this_buffer = NULL; /* Init to silence warning */ + + if (buffers) { /* Not the message buffer */ + this_buffer = devm_kcalloc(dev, bufnum, + sizeof(struct xilly_buffer), + GFP_KERNEL); + if (!this_buffer) + return -ENOMEM; + } + + for (i = 0; i < bufnum; i++) { + /* + * Buffers are expected in descending size order, so there + * is either enough space for this buffer or none at all. + */ + + if ((s->left_of_salami < bytebufsize) && + (s->left_of_salami > 0)) { + dev_err(ep->dev, + "Corrupt buffer allocation in IDT. Aborting.\n"); + return -ENODEV; + } + + if (s->left_of_salami == 0) { + int allocorder, allocsize; + + allocsize = PAGE_SIZE; + allocorder = 0; + while (bytebufsize > allocsize) { + allocsize *= 2; + allocorder++; + } + + s->salami = (void *) devm_get_free_pages( + dev, + GFP_KERNEL | __GFP_DMA32 | __GFP_ZERO, + allocorder); + if (!s->salami) + return -ENOMEM; + + s->left_of_salami = allocsize; + } + + rc = xilly_map_single(ep, s->salami, + bytebufsize, s->direction, + &dma_addr); + if (rc) + return rc; + + iowrite32((u32) (dma_addr & 0xffffffff), + ep->registers + fpga_dma_bufaddr_lowaddr_reg); + iowrite32(((u32) ((((u64) dma_addr) >> 32) & 0xffffffff)), + ep->registers + fpga_dma_bufaddr_highaddr_reg); + + if (buffers) { /* Not the message buffer */ + this_buffer->addr = s->salami; + this_buffer->dma_addr = dma_addr; + buffers[i] = this_buffer++; + + iowrite32(s->regdirection | s->nbuffer++, + ep->registers + fpga_dma_bufno_reg); + } else { + ep->msgbuf_addr = s->salami; + ep->msgbuf_dma_addr = dma_addr; + ep->msg_buf_size = bytebufsize; + + iowrite32(s->regdirection, + ep->registers + fpga_dma_bufno_reg); + } + + s->left_of_salami -= bytebufsize; + s->salami += bytebufsize; + } + return 0; +} + +static int xilly_setupchannels(struct xilly_endpoint *ep, + unsigned char *chandesc, + int entries) +{ + struct device *dev = ep->dev; + int i, entry, rc; + struct xilly_channel *channel; + int channelnum, bufnum, bufsize, format, is_writebuf; + int bytebufsize; + int synchronous, allowpartial, exclusive_open, seekable; + int supports_nonempty; + int msg_buf_done = 0; + + struct xilly_alloc_state rd_alloc = { + .salami = NULL, + .left_of_salami = 0, + .nbuffer = 1, + .direction = DMA_TO_DEVICE, + .regdirection = 0, + }; + + struct xilly_alloc_state wr_alloc = { + .salami = NULL, + .left_of_salami = 0, + .nbuffer = 1, + .direction = DMA_FROM_DEVICE, + .regdirection = 0x80000000, + }; + + channel = devm_kcalloc(dev, ep->num_channels, + sizeof(struct xilly_channel), GFP_KERNEL); + if (!channel) + return -ENOMEM; + + ep->channels = devm_kcalloc(dev, ep->num_channels + 1, + sizeof(struct xilly_channel *), + GFP_KERNEL); + if (!ep->channels) + return -ENOMEM; + + ep->channels[0] = NULL; /* Channel 0 is message buf. */ + + /* Initialize all channels with defaults */ + + for (i = 1; i <= ep->num_channels; i++) { + channel->wr_buffers = NULL; + channel->rd_buffers = NULL; + channel->num_wr_buffers = 0; + channel->num_rd_buffers = 0; + channel->wr_fpga_buf_idx = -1; + channel->wr_host_buf_idx = 0; + channel->wr_host_buf_pos = 0; + channel->wr_empty = 1; + channel->wr_ready = 0; + channel->wr_sleepy = 1; + channel->rd_fpga_buf_idx = 0; + channel->rd_host_buf_idx = 0; + channel->rd_host_buf_pos = 0; + channel->rd_full = 0; + channel->wr_ref_count = 0; + channel->rd_ref_count = 0; + + spin_lock_init(&channel->wr_spinlock); + spin_lock_init(&channel->rd_spinlock); + mutex_init(&channel->wr_mutex); + mutex_init(&channel->rd_mutex); + init_waitqueue_head(&channel->rd_wait); + init_waitqueue_head(&channel->wr_wait); + init_waitqueue_head(&channel->wr_ready_wait); + + INIT_DELAYED_WORK(&channel->rd_workitem, xillybus_autoflush); + + channel->endpoint = ep; + channel->chan_num = i; + + channel->log2_element_size = 0; + + ep->channels[i] = channel++; + } + + for (entry = 0; entry < entries; entry++, chandesc += 4) { + struct xilly_buffer **buffers = NULL; + + is_writebuf = chandesc[0] & 0x01; + channelnum = (chandesc[0] >> 1) | ((chandesc[1] & 0x0f) << 7); + format = (chandesc[1] >> 4) & 0x03; + allowpartial = (chandesc[1] >> 6) & 0x01; + synchronous = (chandesc[1] >> 7) & 0x01; + bufsize = 1 << (chandesc[2] & 0x1f); + bufnum = 1 << (chandesc[3] & 0x0f); + exclusive_open = (chandesc[2] >> 7) & 0x01; + seekable = (chandesc[2] >> 6) & 0x01; + supports_nonempty = (chandesc[2] >> 5) & 0x01; + + if ((channelnum > ep->num_channels) || + ((channelnum == 0) && !is_writebuf)) { + dev_err(ep->dev, + "IDT requests channel out of range. Aborting.\n"); + return -ENODEV; + } + + channel = ep->channels[channelnum]; /* NULL for msg channel */ + + if (!is_writebuf || channelnum > 0) { + channel->log2_element_size = ((format > 2) ? + 2 : format); + + bytebufsize = bufsize * + (1 << channel->log2_element_size); + + buffers = devm_kcalloc(dev, bufnum, + sizeof(struct xilly_buffer *), + GFP_KERNEL); + if (!buffers) + return -ENOMEM; + } else { + bytebufsize = bufsize << 2; + } + + if (!is_writebuf) { + channel->num_rd_buffers = bufnum; + channel->rd_buf_size = bytebufsize; + channel->rd_allow_partial = allowpartial; + channel->rd_synchronous = synchronous; + channel->rd_exclusive_open = exclusive_open; + channel->seekable = seekable; + + channel->rd_buffers = buffers; + rc = xilly_get_dma_buffers(ep, &rd_alloc, buffers, + bufnum, bytebufsize); + } else if (channelnum > 0) { + channel->num_wr_buffers = bufnum; + channel->wr_buf_size = bytebufsize; + + channel->seekable = seekable; + channel->wr_supports_nonempty = supports_nonempty; + + channel->wr_allow_partial = allowpartial; + channel->wr_synchronous = synchronous; + channel->wr_exclusive_open = exclusive_open; + + channel->wr_buffers = buffers; + rc = xilly_get_dma_buffers(ep, &wr_alloc, buffers, + bufnum, bytebufsize); + } else { + rc = xilly_get_dma_buffers(ep, &wr_alloc, NULL, + bufnum, bytebufsize); + msg_buf_done++; + } + + if (rc) + return -ENOMEM; + } + + if (!msg_buf_done) { + dev_err(ep->dev, + "Corrupt IDT: No message buffer. Aborting.\n"); + return -ENODEV; + } + return 0; +} + +static int xilly_scan_idt(struct xilly_endpoint *endpoint, + struct xilly_idt_handle *idt_handle) +{ + int count = 0; + unsigned char *idt = endpoint->channels[1]->wr_buffers[0]->addr; + unsigned char *end_of_idt = idt + endpoint->idtlen - 4; + unsigned char *scan; + int len; + + scan = idt + 1; + idt_handle->names = scan; + + while ((scan <= end_of_idt) && *scan) { + while ((scan <= end_of_idt) && *scan++) + /* Do nothing, just scan thru string */; + count++; + } + + idt_handle->names_len = scan - idt_handle->names; + + scan++; + + if (scan > end_of_idt) { + dev_err(endpoint->dev, + "IDT device name list overflow. Aborting.\n"); + return -ENODEV; + } + idt_handle->chandesc = scan; + + len = endpoint->idtlen - (3 + ((int) (scan - idt))); + + if (len & 0x03) { + dev_err(endpoint->dev, + "Corrupt IDT device name list. Aborting.\n"); + return -ENODEV; + } + + idt_handle->entries = len >> 2; + endpoint->num_channels = count; + + return 0; +} + +static int xilly_obtain_idt(struct xilly_endpoint *endpoint) +{ + struct xilly_channel *channel; + unsigned char *version; + long t; + + channel = endpoint->channels[1]; /* This should be generated ad-hoc */ + + channel->wr_sleepy = 1; + + iowrite32(1 | + (3 << 24), /* Opcode 3 for channel 0 = Send IDT */ + endpoint->registers + fpga_buf_ctrl_reg); + + t = wait_event_interruptible_timeout(channel->wr_wait, + (!channel->wr_sleepy), + XILLY_TIMEOUT); + + if (t <= 0) { + dev_err(endpoint->dev, "Failed to obtain IDT. Aborting.\n"); + + if (endpoint->fatal_error) + return -EIO; + + return -ENODEV; + } + + dma_sync_single_for_cpu(channel->endpoint->dev, + channel->wr_buffers[0]->dma_addr, + channel->wr_buf_size, + DMA_FROM_DEVICE); + + if (channel->wr_buffers[0]->end_offset != endpoint->idtlen) { + dev_err(endpoint->dev, + "IDT length mismatch (%d != %d). Aborting.\n", + channel->wr_buffers[0]->end_offset, endpoint->idtlen); + return -ENODEV; + } + + if (crc32_le(~0, channel->wr_buffers[0]->addr, + endpoint->idtlen+1) != 0) { + dev_err(endpoint->dev, "IDT failed CRC check. Aborting.\n"); + return -ENODEV; + } + + version = channel->wr_buffers[0]->addr; + + /* Check version number. Reject anything above 0x82. */ + if (*version > 0x82) { + dev_err(endpoint->dev, + "No support for IDT version 0x%02x. Maybe the xillybus driver needs an upgrade. Aborting.\n", + *version); + return -ENODEV; + } + + return 0; +} + +static ssize_t xillybus_read(struct file *filp, char __user *userbuf, + size_t count, loff_t *f_pos) +{ + ssize_t rc; + unsigned long flags; + int bytes_done = 0; + int no_time_left = 0; + long deadline, left_to_sleep; + struct xilly_channel *channel = filp->private_data; + + int empty, reached_eof, exhausted, ready; + /* Initializations are there only to silence warnings */ + + int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0; + int waiting_bufidx; + + if (channel->endpoint->fatal_error) + return -EIO; + + deadline = jiffies + 1 + XILLY_RX_TIMEOUT; + + rc = mutex_lock_interruptible(&channel->wr_mutex); + if (rc) + return rc; + + while (1) { /* Note that we may drop mutex within this loop */ + int bytes_to_do = count - bytes_done; + + spin_lock_irqsave(&channel->wr_spinlock, flags); + + empty = channel->wr_empty; + ready = !empty || channel->wr_ready; + + if (!empty) { + bufidx = channel->wr_host_buf_idx; + bufpos = channel->wr_host_buf_pos; + howmany = ((channel->wr_buffers[bufidx]->end_offset + + 1) << channel->log2_element_size) + - bufpos; + + /* Update wr_host_* to its post-operation state */ + if (howmany > bytes_to_do) { + bufferdone = 0; + + howmany = bytes_to_do; + channel->wr_host_buf_pos += howmany; + } else { + bufferdone = 1; + + channel->wr_host_buf_pos = 0; + + if (bufidx == channel->wr_fpga_buf_idx) { + channel->wr_empty = 1; + channel->wr_sleepy = 1; + channel->wr_ready = 0; + } + + if (bufidx >= (channel->num_wr_buffers - 1)) + channel->wr_host_buf_idx = 0; + else + channel->wr_host_buf_idx++; + } + } + + /* + * Marking our situation after the possible changes above, + * for use after releasing the spinlock. + * + * empty = empty before change + * exhasted = empty after possible change + */ + + reached_eof = channel->wr_empty && + (channel->wr_host_buf_idx == channel->wr_eof); + channel->wr_hangup = reached_eof; + exhausted = channel->wr_empty; + waiting_bufidx = channel->wr_host_buf_idx; + + spin_unlock_irqrestore(&channel->wr_spinlock, flags); + + if (!empty) { /* Go on, now without the spinlock */ + + if (bufpos == 0) /* Position zero means it's virgin */ + dma_sync_single_for_cpu(channel->endpoint->dev, + channel->wr_buffers[bufidx]->dma_addr, + channel->wr_buf_size, + DMA_FROM_DEVICE); + + if (copy_to_user( + userbuf, + channel->wr_buffers[bufidx]->addr + + bufpos, howmany)) + rc = -EFAULT; + + userbuf += howmany; + bytes_done += howmany; + + if (bufferdone) { + dma_sync_single_for_device(channel->endpoint->dev, + channel->wr_buffers[bufidx]->dma_addr, + channel->wr_buf_size, + DMA_FROM_DEVICE); + + /* + * Tell FPGA the buffer is done with. It's an + * atomic operation to the FPGA, so what + * happens with other channels doesn't matter, + * and the certain channel is protected with + * the channel-specific mutex. + */ + + iowrite32(1 | (channel->chan_num << 1) | + (bufidx << 12), + channel->endpoint->registers + + fpga_buf_ctrl_reg); + } + + if (rc) { + mutex_unlock(&channel->wr_mutex); + return rc; + } + } + + /* This includes a zero-count return = EOF */ + if ((bytes_done >= count) || reached_eof) + break; + + if (!exhausted) + continue; /* More in RAM buffer(s)? Just go on. */ + + if ((bytes_done > 0) && + (no_time_left || + (channel->wr_synchronous && channel->wr_allow_partial))) + break; + + /* + * Nonblocking read: The "ready" flag tells us that the FPGA + * has data to send. In non-blocking mode, if it isn't on, + * just return. But if there is, we jump directly to the point + * where we ask for the FPGA to send all it has, and wait + * until that data arrives. So in a sense, we *do* block in + * nonblocking mode, but only for a very short time. + */ + + if (!no_time_left && (filp->f_flags & O_NONBLOCK)) { + if (bytes_done > 0) + break; + + if (ready) + goto desperate; + + rc = -EAGAIN; + break; + } + + if (!no_time_left || (bytes_done > 0)) { + /* + * Note that in case of an element-misaligned read + * request, offsetlimit will include the last element, + * which will be partially read from. + */ + int offsetlimit = ((count - bytes_done) - 1) >> + channel->log2_element_size; + int buf_elements = channel->wr_buf_size >> + channel->log2_element_size; + + /* + * In synchronous mode, always send an offset limit. + * Just don't send a value too big. + */ + + if (channel->wr_synchronous) { + /* Don't request more than one buffer */ + if (channel->wr_allow_partial && + (offsetlimit >= buf_elements)) + offsetlimit = buf_elements - 1; + + /* Don't request more than all buffers */ + if (!channel->wr_allow_partial && + (offsetlimit >= + (buf_elements * channel->num_wr_buffers))) + offsetlimit = buf_elements * + channel->num_wr_buffers - 1; + } + + /* + * In asynchronous mode, force early flush of a buffer + * only if that will allow returning a full count. The + * "offsetlimit < ( ... )" rather than "<=" excludes + * requesting a full buffer, which would obviously + * cause a buffer transmission anyhow + */ + + if (channel->wr_synchronous || + (offsetlimit < (buf_elements - 1))) { + mutex_lock(&channel->endpoint->register_mutex); + + iowrite32(offsetlimit, + channel->endpoint->registers + + fpga_buf_offset_reg); + + iowrite32(1 | (channel->chan_num << 1) | + (2 << 24) | /* 2 = offset limit */ + (waiting_bufidx << 12), + channel->endpoint->registers + + fpga_buf_ctrl_reg); + + mutex_unlock(&channel->endpoint-> + register_mutex); + } + } + + /* + * If partial completion is disallowed, there is no point in + * timeout sleeping. Neither if no_time_left is set and + * there's no data. + */ + + if (!channel->wr_allow_partial || + (no_time_left && (bytes_done == 0))) { + /* + * This do-loop will run more than once if another + * thread reasserted wr_sleepy before we got the mutex + * back, so we try again. + */ + + do { + mutex_unlock(&channel->wr_mutex); + + if (wait_event_interruptible( + channel->wr_wait, + (!channel->wr_sleepy))) + goto interrupted; + + if (mutex_lock_interruptible( + &channel->wr_mutex)) + goto interrupted; + } while (channel->wr_sleepy); + + continue; + +interrupted: /* Mutex is not held if got here */ + if (channel->endpoint->fatal_error) + return -EIO; + if (bytes_done) + return bytes_done; + if (filp->f_flags & O_NONBLOCK) + return -EAGAIN; /* Don't admit snoozing */ + return -EINTR; + } + + left_to_sleep = deadline - ((long) jiffies); + + /* + * If our time is out, skip the waiting. We may miss wr_sleepy + * being deasserted but hey, almost missing the train is like + * missing it. + */ + + if (left_to_sleep > 0) { + left_to_sleep = + wait_event_interruptible_timeout( + channel->wr_wait, + (!channel->wr_sleepy), + left_to_sleep); + + if (left_to_sleep > 0) /* wr_sleepy deasserted */ + continue; + + if (left_to_sleep < 0) { /* Interrupt */ + mutex_unlock(&channel->wr_mutex); + if (channel->endpoint->fatal_error) + return -EIO; + if (bytes_done) + return bytes_done; + return -EINTR; + } + } + +desperate: + no_time_left = 1; /* We're out of sleeping time. Desperate! */ + + if (bytes_done == 0) { + /* + * Reaching here means that we allow partial return, + * that we've run out of time, and that we have + * nothing to return. + * So tell the FPGA to send anything it has or gets. + */ + + iowrite32(1 | (channel->chan_num << 1) | + (3 << 24) | /* Opcode 3, flush it all! */ + (waiting_bufidx << 12), + channel->endpoint->registers + + fpga_buf_ctrl_reg); + } + + /* + * Reaching here means that we *do* have data in the buffer, + * but the "partial" flag disallows returning less than + * required. And we don't have as much. So loop again, + * which is likely to end up blocking indefinitely until + * enough data has arrived. + */ + } + + mutex_unlock(&channel->wr_mutex); + + if (channel->endpoint->fatal_error) + return -EIO; + + if (rc) + return rc; + + return bytes_done; +} + +/* + * The timeout argument takes values as follows: + * >0 : Flush with timeout + * ==0 : Flush, and wait idefinitely for the flush to complete + * <0 : Autoflush: Flush only if there's a single buffer occupied + */ + +static int xillybus_myflush(struct xilly_channel *channel, long timeout) +{ + int rc; + unsigned long flags; + + int end_offset_plus1; + int bufidx, bufidx_minus1; + int i; + int empty; + int new_rd_host_buf_pos; + + if (channel->endpoint->fatal_error) + return -EIO; + rc = mutex_lock_interruptible(&channel->rd_mutex); + if (rc) + return rc; + + /* + * Don't flush a closed channel. This can happen when the work queued + * autoflush thread fires off after the file has closed. This is not + * an error, just something to dismiss. + */ + + if (!channel->rd_ref_count) + goto done; + + bufidx = channel->rd_host_buf_idx; + + bufidx_minus1 = (bufidx == 0) ? + channel->num_rd_buffers - 1 : + bufidx - 1; + + end_offset_plus1 = channel->rd_host_buf_pos >> + channel->log2_element_size; + + new_rd_host_buf_pos = channel->rd_host_buf_pos - + (end_offset_plus1 << channel->log2_element_size); + + /* Submit the current buffer if it's nonempty */ + if (end_offset_plus1) { + unsigned char *tail = channel->rd_buffers[bufidx]->addr + + (end_offset_plus1 << channel->log2_element_size); + + /* Copy unflushed data, so we can put it in next buffer */ + for (i = 0; i < new_rd_host_buf_pos; i++) + channel->rd_leftovers[i] = *tail++; + + spin_lock_irqsave(&channel->rd_spinlock, flags); + + /* Autoflush only if a single buffer is occupied */ + + if ((timeout < 0) && + (channel->rd_full || + (bufidx_minus1 != channel->rd_fpga_buf_idx))) { + spin_unlock_irqrestore(&channel->rd_spinlock, flags); + /* + * A new work item may be queued by the ISR exactly + * now, since the execution of a work item allows the + * queuing of a new one while it's running. + */ + goto done; + } + + /* The 4th element is never needed for data, so it's a flag */ + channel->rd_leftovers[3] = (new_rd_host_buf_pos != 0); + + /* Set up rd_full to reflect a certain moment's state */ + + if (bufidx == channel->rd_fpga_buf_idx) + channel->rd_full = 1; + spin_unlock_irqrestore(&channel->rd_spinlock, flags); + + if (bufidx >= (channel->num_rd_buffers - 1)) + channel->rd_host_buf_idx = 0; + else + channel->rd_host_buf_idx++; + + dma_sync_single_for_device(channel->endpoint->dev, + channel->rd_buffers[bufidx]->dma_addr, + channel->rd_buf_size, + DMA_TO_DEVICE); + + mutex_lock(&channel->endpoint->register_mutex); + + iowrite32(end_offset_plus1 - 1, + channel->endpoint->registers + fpga_buf_offset_reg); + + iowrite32((channel->chan_num << 1) | /* Channel ID */ + (2 << 24) | /* Opcode 2, submit buffer */ + (bufidx << 12), + channel->endpoint->registers + fpga_buf_ctrl_reg); + + mutex_unlock(&channel->endpoint->register_mutex); + } else if (bufidx == 0) { + bufidx = channel->num_rd_buffers - 1; + } else { + bufidx--; + } + + channel->rd_host_buf_pos = new_rd_host_buf_pos; + + if (timeout < 0) + goto done; /* Autoflush */ + + /* + * bufidx is now the last buffer written to (or equal to + * rd_fpga_buf_idx if buffer was never written to), and + * channel->rd_host_buf_idx the one after it. + * + * If bufidx == channel->rd_fpga_buf_idx we're either empty or full. + */ + + while (1) { /* Loop waiting for draining of buffers */ + spin_lock_irqsave(&channel->rd_spinlock, flags); + + if (bufidx != channel->rd_fpga_buf_idx) + channel->rd_full = 1; /* + * Not really full, + * but needs waiting. + */ + + empty = !channel->rd_full; + + spin_unlock_irqrestore(&channel->rd_spinlock, flags); + + if (empty) + break; + + /* + * Indefinite sleep with mutex taken. With data waiting for + * flushing user should not be surprised if open() for write + * sleeps. + */ + if (timeout == 0) + wait_event_interruptible(channel->rd_wait, + (!channel->rd_full)); + + else if (wait_event_interruptible_timeout( + channel->rd_wait, + (!channel->rd_full), + timeout) == 0) { + dev_warn(channel->endpoint->dev, + "Timed out while flushing. Output data may be lost.\n"); + + rc = -ETIMEDOUT; + break; + } + + if (channel->rd_full) { + rc = -EINTR; + break; + } + } + +done: + mutex_unlock(&channel->rd_mutex); + + if (channel->endpoint->fatal_error) + return -EIO; + + return rc; +} + +static int xillybus_flush(struct file *filp, fl_owner_t id) +{ + if (!(filp->f_mode & FMODE_WRITE)) + return 0; + + return xillybus_myflush(filp->private_data, HZ); /* 1 second timeout */ +} + +static void xillybus_autoflush(struct work_struct *work) +{ + struct delayed_work *workitem = container_of( + work, struct delayed_work, work); + struct xilly_channel *channel = container_of( + workitem, struct xilly_channel, rd_workitem); + int rc; + + rc = xillybus_myflush(channel, -1); + if (rc == -EINTR) + dev_warn(channel->endpoint->dev, + "Autoflush failed because work queue thread got a signal.\n"); + else if (rc) + dev_err(channel->endpoint->dev, + "Autoflush failed under weird circumstances.\n"); +} + +static ssize_t xillybus_write(struct file *filp, const char __user *userbuf, + size_t count, loff_t *f_pos) +{ + ssize_t rc; + unsigned long flags; + int bytes_done = 0; + struct xilly_channel *channel = filp->private_data; + + int full, exhausted; + /* Initializations are there only to silence warnings */ + + int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0; + int end_offset_plus1 = 0; + + if (channel->endpoint->fatal_error) + return -EIO; + + rc = mutex_lock_interruptible(&channel->rd_mutex); + if (rc) + return rc; + + while (1) { + int bytes_to_do = count - bytes_done; + + spin_lock_irqsave(&channel->rd_spinlock, flags); + + full = channel->rd_full; + + if (!full) { + bufidx = channel->rd_host_buf_idx; + bufpos = channel->rd_host_buf_pos; + howmany = channel->rd_buf_size - bufpos; + + /* + * Update rd_host_* to its state after this operation. + * count=0 means committing the buffer immediately, + * which is like flushing, but not necessarily block. + */ + + if ((howmany > bytes_to_do) && + (count || + ((bufpos >> channel->log2_element_size) == 0))) { + bufferdone = 0; + + howmany = bytes_to_do; + channel->rd_host_buf_pos += howmany; + } else { + bufferdone = 1; + + if (count) { + end_offset_plus1 = + channel->rd_buf_size >> + channel->log2_element_size; + channel->rd_host_buf_pos = 0; + } else { + unsigned char *tail; + int i; + + howmany = 0; + + end_offset_plus1 = bufpos >> + channel->log2_element_size; + + channel->rd_host_buf_pos -= + end_offset_plus1 << + channel->log2_element_size; + + tail = channel-> + rd_buffers[bufidx]->addr + + (end_offset_plus1 << + channel->log2_element_size); + + for (i = 0; + i < channel->rd_host_buf_pos; + i++) + channel->rd_leftovers[i] = + *tail++; + } + + if (bufidx == channel->rd_fpga_buf_idx) + channel->rd_full = 1; + + if (bufidx >= (channel->num_rd_buffers - 1)) + channel->rd_host_buf_idx = 0; + else + channel->rd_host_buf_idx++; + } + } + + /* + * Marking our situation after the possible changes above, + * for use after releasing the spinlock. + * + * full = full before change + * exhasted = full after possible change + */ + + exhausted = channel->rd_full; + + spin_unlock_irqrestore(&channel->rd_spinlock, flags); + + if (!full) { /* Go on, now without the spinlock */ + unsigned char *head = + channel->rd_buffers[bufidx]->addr; + int i; + + if ((bufpos == 0) || /* Zero means it's virgin */ + (channel->rd_leftovers[3] != 0)) { + dma_sync_single_for_cpu(channel->endpoint->dev, + channel->rd_buffers[bufidx]->dma_addr, + channel->rd_buf_size, + DMA_TO_DEVICE); + + /* Virgin, but leftovers are due */ + for (i = 0; i < bufpos; i++) + *head++ = channel->rd_leftovers[i]; + + channel->rd_leftovers[3] = 0; /* Clear flag */ + } + + if (copy_from_user( + channel->rd_buffers[bufidx]->addr + bufpos, + userbuf, howmany)) + rc = -EFAULT; + + userbuf += howmany; + bytes_done += howmany; + + if (bufferdone) { + dma_sync_single_for_device(channel->endpoint->dev, + channel->rd_buffers[bufidx]->dma_addr, + channel->rd_buf_size, + DMA_TO_DEVICE); + + mutex_lock(&channel->endpoint->register_mutex); + + iowrite32(end_offset_plus1 - 1, + channel->endpoint->registers + + fpga_buf_offset_reg); + + iowrite32((channel->chan_num << 1) | + (2 << 24) | /* 2 = submit buffer */ + (bufidx << 12), + channel->endpoint->registers + + fpga_buf_ctrl_reg); + + mutex_unlock(&channel->endpoint-> + register_mutex); + + channel->rd_leftovers[3] = + (channel->rd_host_buf_pos != 0); + } + + if (rc) { + mutex_unlock(&channel->rd_mutex); + + if (channel->endpoint->fatal_error) + return -EIO; + + if (!channel->rd_synchronous) + queue_delayed_work( + xillybus_wq, + &channel->rd_workitem, + XILLY_RX_TIMEOUT); + + return rc; + } + } + + if (bytes_done >= count) + break; + + if (!exhausted) + continue; /* If there's more space, just go on */ + + if ((bytes_done > 0) && channel->rd_allow_partial) + break; + + /* + * Indefinite sleep with mutex taken. With data waiting for + * flushing, user should not be surprised if open() for write + * sleeps. + */ + + if (filp->f_flags & O_NONBLOCK) { + rc = -EAGAIN; + break; + } + + if (wait_event_interruptible(channel->rd_wait, + (!channel->rd_full))) { + mutex_unlock(&channel->rd_mutex); + + if (channel->endpoint->fatal_error) + return -EIO; + + if (bytes_done) + return bytes_done; + return -EINTR; + } + } + + mutex_unlock(&channel->rd_mutex); + + if (!channel->rd_synchronous) + queue_delayed_work(xillybus_wq, + &channel->rd_workitem, + XILLY_RX_TIMEOUT); + + if (channel->endpoint->fatal_error) + return -EIO; + + if (rc) + return rc; + + if ((channel->rd_synchronous) && (bytes_done > 0)) { + rc = xillybus_myflush(filp->private_data, 0); /* No timeout */ + + if (rc && (rc != -EINTR)) + return rc; + } + + return bytes_done; +} + +static int xillybus_open(struct inode *inode, struct file *filp) +{ + int rc; + unsigned long flags; + struct xilly_endpoint *endpoint; + struct xilly_channel *channel; + int index; + + rc = xillybus_find_inode(inode, (void **)&endpoint, &index); + if (rc) + return rc; + + if (endpoint->fatal_error) + return -EIO; + + channel = endpoint->channels[1 + index]; + filp->private_data = channel; + + /* + * It gets complicated because: + * 1. We don't want to take a mutex we don't have to + * 2. We don't want to open one direction if the other will fail. + */ + + if ((filp->f_mode & FMODE_READ) && (!channel->num_wr_buffers)) + return -ENODEV; + + if ((filp->f_mode & FMODE_WRITE) && (!channel->num_rd_buffers)) + return -ENODEV; + + if ((filp->f_mode & FMODE_READ) && (filp->f_flags & O_NONBLOCK) && + (channel->wr_synchronous || !channel->wr_allow_partial || + !channel->wr_supports_nonempty)) { + dev_err(endpoint->dev, + "open() failed: O_NONBLOCK not allowed for read on this device\n"); + return -ENODEV; + } + + if ((filp->f_mode & FMODE_WRITE) && (filp->f_flags & O_NONBLOCK) && + (channel->rd_synchronous || !channel->rd_allow_partial)) { + dev_err(endpoint->dev, + "open() failed: O_NONBLOCK not allowed for write on this device\n"); + return -ENODEV; + } + + /* + * Note: open() may block on getting mutexes despite O_NONBLOCK. + * This shouldn't occur normally, since multiple open of the same + * file descriptor is almost always prohibited anyhow + * (*_exclusive_open is normally set in real-life systems). + */ + + if (filp->f_mode & FMODE_READ) { + rc = mutex_lock_interruptible(&channel->wr_mutex); + if (rc) + return rc; + } + + if (filp->f_mode & FMODE_WRITE) { + rc = mutex_lock_interruptible(&channel->rd_mutex); + if (rc) + goto unlock_wr; + } + + if ((filp->f_mode & FMODE_READ) && + (channel->wr_ref_count != 0) && + (channel->wr_exclusive_open)) { + rc = -EBUSY; + goto unlock; + } + + if ((filp->f_mode & FMODE_WRITE) && + (channel->rd_ref_count != 0) && + (channel->rd_exclusive_open)) { + rc = -EBUSY; + goto unlock; + } + + if (filp->f_mode & FMODE_READ) { + if (channel->wr_ref_count == 0) { /* First open of file */ + /* Move the host to first buffer */ + spin_lock_irqsave(&channel->wr_spinlock, flags); + channel->wr_host_buf_idx = 0; + channel->wr_host_buf_pos = 0; + channel->wr_fpga_buf_idx = -1; + channel->wr_empty = 1; + channel->wr_ready = 0; + channel->wr_sleepy = 1; + channel->wr_eof = -1; + channel->wr_hangup = 0; + + spin_unlock_irqrestore(&channel->wr_spinlock, flags); + + iowrite32(1 | (channel->chan_num << 1) | + (4 << 24) | /* Opcode 4, open channel */ + ((channel->wr_synchronous & 1) << 23), + channel->endpoint->registers + + fpga_buf_ctrl_reg); + } + + channel->wr_ref_count++; + } + + if (filp->f_mode & FMODE_WRITE) { + if (channel->rd_ref_count == 0) { /* First open of file */ + /* Move the host to first buffer */ + spin_lock_irqsave(&channel->rd_spinlock, flags); + channel->rd_host_buf_idx = 0; + channel->rd_host_buf_pos = 0; + channel->rd_leftovers[3] = 0; /* No leftovers. */ + channel->rd_fpga_buf_idx = channel->num_rd_buffers - 1; + channel->rd_full = 0; + + spin_unlock_irqrestore(&channel->rd_spinlock, flags); + + iowrite32((channel->chan_num << 1) | + (4 << 24), /* Opcode 4, open channel */ + channel->endpoint->registers + + fpga_buf_ctrl_reg); + } + + channel->rd_ref_count++; + } + +unlock: + if (filp->f_mode & FMODE_WRITE) + mutex_unlock(&channel->rd_mutex); +unlock_wr: + if (filp->f_mode & FMODE_READ) + mutex_unlock(&channel->wr_mutex); + + if (!rc && (!channel->seekable)) + return nonseekable_open(inode, filp); + + return rc; +} + +static int xillybus_release(struct inode *inode, struct file *filp) +{ + unsigned long flags; + struct xilly_channel *channel = filp->private_data; + + int buf_idx; + int eof; + + if (channel->endpoint->fatal_error) + return -EIO; + + if (filp->f_mode & FMODE_WRITE) { + mutex_lock(&channel->rd_mutex); + + channel->rd_ref_count--; + + if (channel->rd_ref_count == 0) { + /* + * We rely on the kernel calling flush() + * before we get here. + */ + + iowrite32((channel->chan_num << 1) | /* Channel ID */ + (5 << 24), /* Opcode 5, close channel */ + channel->endpoint->registers + + fpga_buf_ctrl_reg); + } + mutex_unlock(&channel->rd_mutex); + } + + if (filp->f_mode & FMODE_READ) { + mutex_lock(&channel->wr_mutex); + + channel->wr_ref_count--; + + if (channel->wr_ref_count == 0) { + iowrite32(1 | (channel->chan_num << 1) | + (5 << 24), /* Opcode 5, close channel */ + channel->endpoint->registers + + fpga_buf_ctrl_reg); + + /* + * This is crazily cautious: We make sure that not + * only that we got an EOF (be it because we closed + * the channel or because of a user's EOF), but verify + * that it's one beyond the last buffer arrived, so + * we have no leftover buffers pending before wrapping + * up (which can only happen in asynchronous channels, + * BTW) + */ + + while (1) { + spin_lock_irqsave(&channel->wr_spinlock, + flags); + buf_idx = channel->wr_fpga_buf_idx; + eof = channel->wr_eof; + channel->wr_sleepy = 1; + spin_unlock_irqrestore(&channel->wr_spinlock, + flags); + + /* + * Check if eof points at the buffer after + * the last one the FPGA submitted. Note that + * no EOF is marked by negative eof. + */ + + buf_idx++; + if (buf_idx == channel->num_wr_buffers) + buf_idx = 0; + + if (buf_idx == eof) + break; + + /* + * Steal extra 100 ms if awaken by interrupt. + * This is a simple workaround for an + * interrupt pending when entering, which would + * otherwise result in declaring the hardware + * non-responsive. + */ + + if (wait_event_interruptible( + channel->wr_wait, + (!channel->wr_sleepy))) + msleep(100); + + if (channel->wr_sleepy) { + mutex_unlock(&channel->wr_mutex); + dev_warn(channel->endpoint->dev, + "Hardware failed to respond to close command, therefore left in messy state.\n"); + return -EINTR; + } + } + } + + mutex_unlock(&channel->wr_mutex); + } + + return 0; +} + +static loff_t xillybus_llseek(struct file *filp, loff_t offset, int whence) +{ + struct xilly_channel *channel = filp->private_data; + loff_t pos = filp->f_pos; + int rc = 0; + + /* + * Take both mutexes not allowing interrupts, since it seems like + * common applications don't expect an -EINTR here. Besides, multiple + * access to a single file descriptor on seekable devices is a mess + * anyhow. + */ + + if (channel->endpoint->fatal_error) + return -EIO; + + mutex_lock(&channel->wr_mutex); + mutex_lock(&channel->rd_mutex); + + switch (whence) { + case SEEK_SET: + pos = offset; + break; + case SEEK_CUR: + pos += offset; + break; + case SEEK_END: + pos = offset; /* Going to the end => to the beginning */ + break; + default: + rc = -EINVAL; + goto end; + } + + /* In any case, we must finish on an element boundary */ + if (pos & ((1 << channel->log2_element_size) - 1)) { + rc = -EINVAL; + goto end; + } + + mutex_lock(&channel->endpoint->register_mutex); + + iowrite32(pos >> channel->log2_element_size, + channel->endpoint->registers + fpga_buf_offset_reg); + + iowrite32((channel->chan_num << 1) | + (6 << 24), /* Opcode 6, set address */ + channel->endpoint->registers + fpga_buf_ctrl_reg); + + mutex_unlock(&channel->endpoint->register_mutex); + +end: + mutex_unlock(&channel->rd_mutex); + mutex_unlock(&channel->wr_mutex); + + if (rc) /* Return error after releasing mutexes */ + return rc; + + filp->f_pos = pos; + + /* + * Since seekable devices are allowed only when the channel is + * synchronous, we assume that there is no data pending in either + * direction (which holds true as long as no concurrent access on the + * file descriptor takes place). + * The only thing we may need to throw away is leftovers from partial + * write() flush. + */ + + channel->rd_leftovers[3] = 0; + + return pos; +} + +static __poll_t xillybus_poll(struct file *filp, poll_table *wait) +{ + struct xilly_channel *channel = filp->private_data; + __poll_t mask = 0; + unsigned long flags; + + poll_wait(filp, &channel->endpoint->ep_wait, wait); + + /* + * poll() won't play ball regarding read() channels which + * aren't asynchronous and support the nonempty message. Allowing + * that will create situations where data has been delivered at + * the FPGA, and users expecting select() to wake up, which it may + * not. + */ + + if (!channel->wr_synchronous && channel->wr_supports_nonempty) { + poll_wait(filp, &channel->wr_wait, wait); + poll_wait(filp, &channel->wr_ready_wait, wait); + + spin_lock_irqsave(&channel->wr_spinlock, flags); + if (!channel->wr_empty || channel->wr_ready) + mask |= EPOLLIN | EPOLLRDNORM; + + if (channel->wr_hangup) + /* + * Not EPOLLHUP, because its behavior is in the + * mist, and EPOLLIN does what we want: Wake up + * the read file descriptor so it sees EOF. + */ + mask |= EPOLLIN | EPOLLRDNORM; + spin_unlock_irqrestore(&channel->wr_spinlock, flags); + } + + /* + * If partial data write is disallowed on a write() channel, + * it's pointless to ever signal OK to write, because is could + * block despite some space being available. + */ + + if (channel->rd_allow_partial) { + poll_wait(filp, &channel->rd_wait, wait); + + spin_lock_irqsave(&channel->rd_spinlock, flags); + if (!channel->rd_full) + mask |= EPOLLOUT | EPOLLWRNORM; + spin_unlock_irqrestore(&channel->rd_spinlock, flags); + } + + if (channel->endpoint->fatal_error) + mask |= EPOLLERR; + + return mask; +} + +static const struct file_operations xillybus_fops = { + .owner = THIS_MODULE, + .read = xillybus_read, + .write = xillybus_write, + .open = xillybus_open, + .flush = xillybus_flush, + .release = xillybus_release, + .llseek = xillybus_llseek, + .poll = xillybus_poll, +}; + +struct xilly_endpoint *xillybus_init_endpoint(struct device *dev) +{ + struct xilly_endpoint *endpoint; + + endpoint = devm_kzalloc(dev, sizeof(*endpoint), GFP_KERNEL); + if (!endpoint) + return NULL; + + endpoint->dev = dev; + endpoint->msg_counter = 0x0b; + endpoint->failed_messages = 0; + endpoint->fatal_error = 0; + + init_waitqueue_head(&endpoint->ep_wait); + mutex_init(&endpoint->register_mutex); + + return endpoint; +} +EXPORT_SYMBOL(xillybus_init_endpoint); + +static int xilly_quiesce(struct xilly_endpoint *endpoint) +{ + long t; + + endpoint->idtlen = -1; + + iowrite32((u32) (endpoint->dma_using_dac & 0x0001), + endpoint->registers + fpga_dma_control_reg); + + t = wait_event_interruptible_timeout(endpoint->ep_wait, + (endpoint->idtlen >= 0), + XILLY_TIMEOUT); + if (t <= 0) { + dev_err(endpoint->dev, + "Failed to quiesce the device on exit.\n"); + return -ENODEV; + } + return 0; +} + +int xillybus_endpoint_discovery(struct xilly_endpoint *endpoint) +{ + int rc; + long t; + + void *bootstrap_resources; + int idtbuffersize = (1 << PAGE_SHIFT); + struct device *dev = endpoint->dev; + + /* + * The bogus IDT is used during bootstrap for allocating the initial + * message buffer, and then the message buffer and space for the IDT + * itself. The initial message buffer is of a single page's size, but + * it's soon replaced with a more modest one (and memory is freed). + */ + + unsigned char bogus_idt[8] = { 1, 224, (PAGE_SHIFT)-2, 0, + 3, 192, PAGE_SHIFT, 0 }; + struct xilly_idt_handle idt_handle; + + /* + * Writing the value 0x00000001 to Endianness register signals which + * endianness this processor is using, so the FPGA can swap words as + * necessary. + */ + + iowrite32(1, endpoint->registers + fpga_endian_reg); + + /* Bootstrap phase I: Allocate temporary message buffer */ + + bootstrap_resources = devres_open_group(dev, NULL, GFP_KERNEL); + if (!bootstrap_resources) + return -ENOMEM; + + endpoint->num_channels = 0; + + rc = xilly_setupchannels(endpoint, bogus_idt, 1); + if (rc) + return rc; + + /* Clear the message subsystem (and counter in particular) */ + iowrite32(0x04, endpoint->registers + fpga_msg_ctrl_reg); + + endpoint->idtlen = -1; + + /* + * Set DMA 32/64 bit mode, quiesce the device (?!) and get IDT + * buffer size. + */ + iowrite32((u32) (endpoint->dma_using_dac & 0x0001), + endpoint->registers + fpga_dma_control_reg); + + t = wait_event_interruptible_timeout(endpoint->ep_wait, + (endpoint->idtlen >= 0), + XILLY_TIMEOUT); + if (t <= 0) { + dev_err(endpoint->dev, "No response from FPGA. Aborting.\n"); + return -ENODEV; + } + + /* Enable DMA */ + iowrite32((u32) (0x0002 | (endpoint->dma_using_dac & 0x0001)), + endpoint->registers + fpga_dma_control_reg); + + /* Bootstrap phase II: Allocate buffer for IDT and obtain it */ + while (endpoint->idtlen >= idtbuffersize) { + idtbuffersize *= 2; + bogus_idt[6]++; + } + + endpoint->num_channels = 1; + + rc = xilly_setupchannels(endpoint, bogus_idt, 2); + if (rc) + goto failed_idt; + + rc = xilly_obtain_idt(endpoint); + if (rc) + goto failed_idt; + + rc = xilly_scan_idt(endpoint, &idt_handle); + if (rc) + goto failed_idt; + + devres_close_group(dev, bootstrap_resources); + + /* Bootstrap phase III: Allocate buffers according to IDT */ + + rc = xilly_setupchannels(endpoint, + idt_handle.chandesc, + idt_handle.entries); + if (rc) + goto failed_idt; + + rc = xillybus_init_chrdev(dev, &xillybus_fops, + endpoint->owner, endpoint, + idt_handle.names, + idt_handle.names_len, + endpoint->num_channels, + xillyname, false); + + if (rc) + goto failed_idt; + + devres_release_group(dev, bootstrap_resources); + + return 0; + +failed_idt: + xilly_quiesce(endpoint); + flush_workqueue(xillybus_wq); + + return rc; +} +EXPORT_SYMBOL(xillybus_endpoint_discovery); + +void xillybus_endpoint_remove(struct xilly_endpoint *endpoint) +{ + xillybus_cleanup_chrdev(endpoint, endpoint->dev); + + xilly_quiesce(endpoint); + + /* + * Flushing is done upon endpoint release to prevent access to memory + * just about to be released. This makes the quiesce complete. + */ + flush_workqueue(xillybus_wq); +} +EXPORT_SYMBOL(xillybus_endpoint_remove); + +static int __init xillybus_init(void) +{ + xillybus_wq = alloc_workqueue(xillyname, 0, 0); + if (!xillybus_wq) + return -ENOMEM; + + return 0; +} + +static void __exit xillybus_exit(void) +{ + /* flush_workqueue() was called for each endpoint released */ + destroy_workqueue(xillybus_wq); +} + +module_init(xillybus_init); +module_exit(xillybus_exit); |