diff options
Diffstat (limited to 'drivers/rapidio/devices/tsi721_dma.c')
| -rw-r--r-- | drivers/rapidio/devices/tsi721_dma.c | 1042 |
1 files changed, 1042 insertions, 0 deletions
diff --git a/drivers/rapidio/devices/tsi721_dma.c b/drivers/rapidio/devices/tsi721_dma.c new file mode 100644 index 000000000..d375c0205 --- /dev/null +++ b/drivers/rapidio/devices/tsi721_dma.c @@ -0,0 +1,1042 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * DMA Engine support for Tsi721 PCIExpress-to-SRIO bridge + * + * Copyright (c) 2011-2014 Integrated Device Technology, Inc. + * Alexandre Bounine <alexandre.bounine@idt.com> + */ + +#include <linux/io.h> +#include <linux/errno.h> +#include <linux/init.h> +#include <linux/ioport.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/rio.h> +#include <linux/rio_drv.h> +#include <linux/dma-mapping.h> +#include <linux/interrupt.h> +#include <linux/kfifo.h> +#include <linux/sched.h> +#include <linux/delay.h> +#include "../../dma/dmaengine.h" + +#include "tsi721.h" + +#ifdef CONFIG_PCI_MSI +static irqreturn_t tsi721_bdma_msix(int irq, void *ptr); +#endif +static int tsi721_submit_sg(struct tsi721_tx_desc *desc); + +static unsigned int dma_desc_per_channel = 128; +module_param(dma_desc_per_channel, uint, S_IRUGO); +MODULE_PARM_DESC(dma_desc_per_channel, + "Number of DMA descriptors per channel (default: 128)"); + +static unsigned int dma_txqueue_sz = 16; +module_param(dma_txqueue_sz, uint, S_IRUGO); +MODULE_PARM_DESC(dma_txqueue_sz, + "DMA Transactions Queue Size (default: 16)"); + +static u8 dma_sel = 0x7f; +module_param(dma_sel, byte, S_IRUGO); +MODULE_PARM_DESC(dma_sel, + "DMA Channel Selection Mask (default: 0x7f = all)"); + +static inline struct tsi721_bdma_chan *to_tsi721_chan(struct dma_chan *chan) +{ + return container_of(chan, struct tsi721_bdma_chan, dchan); +} + +static inline struct tsi721_device *to_tsi721(struct dma_device *ddev) +{ + return container_of(ddev, struct rio_mport, dma)->priv; +} + +static inline +struct tsi721_tx_desc *to_tsi721_desc(struct dma_async_tx_descriptor *txd) +{ + return container_of(txd, struct tsi721_tx_desc, txd); +} + +static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan, int bd_num) +{ + struct tsi721_dma_desc *bd_ptr; + struct device *dev = bdma_chan->dchan.device->dev; + u64 *sts_ptr; + dma_addr_t bd_phys; + dma_addr_t sts_phys; + int sts_size; +#ifdef CONFIG_PCI_MSI + struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device); +#endif + + tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d", bdma_chan->id); + + /* + * Allocate space for DMA descriptors + * (add an extra element for link descriptor) + */ + bd_ptr = dma_alloc_coherent(dev, + (bd_num + 1) * sizeof(struct tsi721_dma_desc), + &bd_phys, GFP_ATOMIC); + if (!bd_ptr) + return -ENOMEM; + + bdma_chan->bd_num = bd_num; + bdma_chan->bd_phys = bd_phys; + bdma_chan->bd_base = bd_ptr; + + tsi_debug(DMA, &bdma_chan->dchan.dev->device, + "DMAC%d descriptors @ %p (phys = %pad)", + bdma_chan->id, bd_ptr, &bd_phys); + + /* Allocate space for descriptor status FIFO */ + sts_size = ((bd_num + 1) >= TSI721_DMA_MINSTSSZ) ? + (bd_num + 1) : TSI721_DMA_MINSTSSZ; + sts_size = roundup_pow_of_two(sts_size); + sts_ptr = dma_alloc_coherent(dev, + sts_size * sizeof(struct tsi721_dma_sts), + &sts_phys, GFP_ATOMIC); + if (!sts_ptr) { + /* Free space allocated for DMA descriptors */ + dma_free_coherent(dev, + (bd_num + 1) * sizeof(struct tsi721_dma_desc), + bd_ptr, bd_phys); + bdma_chan->bd_base = NULL; + return -ENOMEM; + } + + bdma_chan->sts_phys = sts_phys; + bdma_chan->sts_base = sts_ptr; + bdma_chan->sts_size = sts_size; + + tsi_debug(DMA, &bdma_chan->dchan.dev->device, + "DMAC%d desc status FIFO @ %p (phys = %pad) size=0x%x", + bdma_chan->id, sts_ptr, &sts_phys, sts_size); + + /* Initialize DMA descriptors ring using added link descriptor */ + bd_ptr[bd_num].type_id = cpu_to_le32(DTYPE3 << 29); + bd_ptr[bd_num].next_lo = cpu_to_le32((u64)bd_phys & + TSI721_DMAC_DPTRL_MASK); + bd_ptr[bd_num].next_hi = cpu_to_le32((u64)bd_phys >> 32); + + /* Setup DMA descriptor pointers */ + iowrite32(((u64)bd_phys >> 32), + bdma_chan->regs + TSI721_DMAC_DPTRH); + iowrite32(((u64)bd_phys & TSI721_DMAC_DPTRL_MASK), + bdma_chan->regs + TSI721_DMAC_DPTRL); + + /* Setup descriptor status FIFO */ + iowrite32(((u64)sts_phys >> 32), + bdma_chan->regs + TSI721_DMAC_DSBH); + iowrite32(((u64)sts_phys & TSI721_DMAC_DSBL_MASK), + bdma_chan->regs + TSI721_DMAC_DSBL); + iowrite32(TSI721_DMAC_DSSZ_SIZE(sts_size), + bdma_chan->regs + TSI721_DMAC_DSSZ); + + /* Clear interrupt bits */ + iowrite32(TSI721_DMAC_INT_ALL, + bdma_chan->regs + TSI721_DMAC_INT); + + ioread32(bdma_chan->regs + TSI721_DMAC_INT); + +#ifdef CONFIG_PCI_MSI + /* Request interrupt service if we are in MSI-X mode */ + if (priv->flags & TSI721_USING_MSIX) { + int rc, idx; + + idx = TSI721_VECT_DMA0_DONE + bdma_chan->id; + + rc = request_irq(priv->msix[idx].vector, tsi721_bdma_msix, 0, + priv->msix[idx].irq_name, (void *)bdma_chan); + + if (rc) { + tsi_debug(DMA, &bdma_chan->dchan.dev->device, + "Unable to get MSI-X for DMAC%d-DONE", + bdma_chan->id); + goto err_out; + } + + idx = TSI721_VECT_DMA0_INT + bdma_chan->id; + + rc = request_irq(priv->msix[idx].vector, tsi721_bdma_msix, 0, + priv->msix[idx].irq_name, (void *)bdma_chan); + + if (rc) { + tsi_debug(DMA, &bdma_chan->dchan.dev->device, + "Unable to get MSI-X for DMAC%d-INT", + bdma_chan->id); + free_irq( + priv->msix[TSI721_VECT_DMA0_DONE + + bdma_chan->id].vector, + (void *)bdma_chan); + } + +err_out: + if (rc) { + /* Free space allocated for DMA descriptors */ + dma_free_coherent(dev, + (bd_num + 1) * sizeof(struct tsi721_dma_desc), + bd_ptr, bd_phys); + bdma_chan->bd_base = NULL; + + /* Free space allocated for status descriptors */ + dma_free_coherent(dev, + sts_size * sizeof(struct tsi721_dma_sts), + sts_ptr, sts_phys); + bdma_chan->sts_base = NULL; + + return -EIO; + } + } +#endif /* CONFIG_PCI_MSI */ + + /* Toggle DMA channel initialization */ + iowrite32(TSI721_DMAC_CTL_INIT, bdma_chan->regs + TSI721_DMAC_CTL); + ioread32(bdma_chan->regs + TSI721_DMAC_CTL); + bdma_chan->wr_count = bdma_chan->wr_count_next = 0; + bdma_chan->sts_rdptr = 0; + udelay(10); + + return 0; +} + +static int tsi721_bdma_ch_free(struct tsi721_bdma_chan *bdma_chan) +{ + u32 ch_stat; +#ifdef CONFIG_PCI_MSI + struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device); +#endif + + if (!bdma_chan->bd_base) + return 0; + + /* Check if DMA channel still running */ + ch_stat = ioread32(bdma_chan->regs + TSI721_DMAC_STS); + if (ch_stat & TSI721_DMAC_STS_RUN) + return -EFAULT; + + /* Put DMA channel into init state */ + iowrite32(TSI721_DMAC_CTL_INIT, bdma_chan->regs + TSI721_DMAC_CTL); + +#ifdef CONFIG_PCI_MSI + if (priv->flags & TSI721_USING_MSIX) { + free_irq(priv->msix[TSI721_VECT_DMA0_DONE + + bdma_chan->id].vector, (void *)bdma_chan); + free_irq(priv->msix[TSI721_VECT_DMA0_INT + + bdma_chan->id].vector, (void *)bdma_chan); + } +#endif /* CONFIG_PCI_MSI */ + + /* Free space allocated for DMA descriptors */ + dma_free_coherent(bdma_chan->dchan.device->dev, + (bdma_chan->bd_num + 1) * sizeof(struct tsi721_dma_desc), + bdma_chan->bd_base, bdma_chan->bd_phys); + bdma_chan->bd_base = NULL; + + /* Free space allocated for status FIFO */ + dma_free_coherent(bdma_chan->dchan.device->dev, + bdma_chan->sts_size * sizeof(struct tsi721_dma_sts), + bdma_chan->sts_base, bdma_chan->sts_phys); + bdma_chan->sts_base = NULL; + return 0; +} + +static void +tsi721_bdma_interrupt_enable(struct tsi721_bdma_chan *bdma_chan, int enable) +{ + if (enable) { + /* Clear pending BDMA channel interrupts */ + iowrite32(TSI721_DMAC_INT_ALL, + bdma_chan->regs + TSI721_DMAC_INT); + ioread32(bdma_chan->regs + TSI721_DMAC_INT); + /* Enable BDMA channel interrupts */ + iowrite32(TSI721_DMAC_INT_ALL, + bdma_chan->regs + TSI721_DMAC_INTE); + } else { + /* Disable BDMA channel interrupts */ + iowrite32(0, bdma_chan->regs + TSI721_DMAC_INTE); + /* Clear pending BDMA channel interrupts */ + iowrite32(TSI721_DMAC_INT_ALL, + bdma_chan->regs + TSI721_DMAC_INT); + } + +} + +static bool tsi721_dma_is_idle(struct tsi721_bdma_chan *bdma_chan) +{ + u32 sts; + + sts = ioread32(bdma_chan->regs + TSI721_DMAC_STS); + return ((sts & TSI721_DMAC_STS_RUN) == 0); +} + +void tsi721_bdma_handler(struct tsi721_bdma_chan *bdma_chan) +{ + /* Disable BDMA channel interrupts */ + iowrite32(0, bdma_chan->regs + TSI721_DMAC_INTE); + if (bdma_chan->active) + tasklet_hi_schedule(&bdma_chan->tasklet); +} + +#ifdef CONFIG_PCI_MSI +/** + * tsi721_omsg_msix - MSI-X interrupt handler for BDMA channels + * @irq: Linux interrupt number + * @ptr: Pointer to interrupt-specific data (BDMA channel structure) + * + * Handles BDMA channel interrupts signaled using MSI-X. + */ +static irqreturn_t tsi721_bdma_msix(int irq, void *ptr) +{ + struct tsi721_bdma_chan *bdma_chan = ptr; + + if (bdma_chan->active) + tasklet_hi_schedule(&bdma_chan->tasklet); + return IRQ_HANDLED; +} +#endif /* CONFIG_PCI_MSI */ + +/* Must be called with the spinlock held */ +static void tsi721_start_dma(struct tsi721_bdma_chan *bdma_chan) +{ + if (!tsi721_dma_is_idle(bdma_chan)) { + tsi_err(&bdma_chan->dchan.dev->device, + "DMAC%d Attempt to start non-idle channel", + bdma_chan->id); + return; + } + + if (bdma_chan->wr_count == bdma_chan->wr_count_next) { + tsi_err(&bdma_chan->dchan.dev->device, + "DMAC%d Attempt to start DMA with no BDs ready %d", + bdma_chan->id, task_pid_nr(current)); + return; + } + + tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d (wrc=%d) %d", + bdma_chan->id, bdma_chan->wr_count_next, + task_pid_nr(current)); + + iowrite32(bdma_chan->wr_count_next, + bdma_chan->regs + TSI721_DMAC_DWRCNT); + ioread32(bdma_chan->regs + TSI721_DMAC_DWRCNT); + + bdma_chan->wr_count = bdma_chan->wr_count_next; +} + +static int +tsi721_desc_fill_init(struct tsi721_tx_desc *desc, + struct tsi721_dma_desc *bd_ptr, + struct scatterlist *sg, u32 sys_size) +{ + u64 rio_addr; + + if (!bd_ptr) + return -EINVAL; + + /* Initialize DMA descriptor */ + bd_ptr->type_id = cpu_to_le32((DTYPE1 << 29) | + (desc->rtype << 19) | desc->destid); + bd_ptr->bcount = cpu_to_le32(((desc->rio_addr & 0x3) << 30) | + (sys_size << 26)); + rio_addr = (desc->rio_addr >> 2) | + ((u64)(desc->rio_addr_u & 0x3) << 62); + bd_ptr->raddr_lo = cpu_to_le32(rio_addr & 0xffffffff); + bd_ptr->raddr_hi = cpu_to_le32(rio_addr >> 32); + bd_ptr->t1.bufptr_lo = cpu_to_le32( + (u64)sg_dma_address(sg) & 0xffffffff); + bd_ptr->t1.bufptr_hi = cpu_to_le32((u64)sg_dma_address(sg) >> 32); + bd_ptr->t1.s_dist = 0; + bd_ptr->t1.s_size = 0; + + return 0; +} + +static int +tsi721_desc_fill_end(struct tsi721_dma_desc *bd_ptr, u32 bcount, bool interrupt) +{ + if (!bd_ptr) + return -EINVAL; + + /* Update DMA descriptor */ + if (interrupt) + bd_ptr->type_id |= cpu_to_le32(TSI721_DMAD_IOF); + bd_ptr->bcount |= cpu_to_le32(bcount & TSI721_DMAD_BCOUNT1); + + return 0; +} + +static void tsi721_dma_tx_err(struct tsi721_bdma_chan *bdma_chan, + struct tsi721_tx_desc *desc) +{ + struct dma_async_tx_descriptor *txd = &desc->txd; + dma_async_tx_callback callback = txd->callback; + void *param = txd->callback_param; + + list_move(&desc->desc_node, &bdma_chan->free_list); + + if (callback) + callback(param); +} + +static void tsi721_clr_stat(struct tsi721_bdma_chan *bdma_chan) +{ + u32 srd_ptr; + u64 *sts_ptr; + int i, j; + + /* Check and clear descriptor status FIFO entries */ + srd_ptr = bdma_chan->sts_rdptr; + sts_ptr = bdma_chan->sts_base; + j = srd_ptr * 8; + while (sts_ptr[j]) { + for (i = 0; i < 8 && sts_ptr[j]; i++, j++) + sts_ptr[j] = 0; + + ++srd_ptr; + srd_ptr %= bdma_chan->sts_size; + j = srd_ptr * 8; + } + + iowrite32(srd_ptr, bdma_chan->regs + TSI721_DMAC_DSRP); + bdma_chan->sts_rdptr = srd_ptr; +} + +/* Must be called with the channel spinlock held */ +static int tsi721_submit_sg(struct tsi721_tx_desc *desc) +{ + struct dma_chan *dchan = desc->txd.chan; + struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); + u32 sys_size; + u64 rio_addr; + dma_addr_t next_addr; + u32 bcount; + struct scatterlist *sg; + unsigned int i; + int err = 0; + struct tsi721_dma_desc *bd_ptr = NULL; + u32 idx, rd_idx; + u32 add_count = 0; + struct device *ch_dev = &dchan->dev->device; + + if (!tsi721_dma_is_idle(bdma_chan)) { + tsi_err(ch_dev, "DMAC%d ERR: Attempt to use non-idle channel", + bdma_chan->id); + return -EIO; + } + + /* + * Fill DMA channel's hardware buffer descriptors. + * (NOTE: RapidIO destination address is limited to 64 bits for now) + */ + rio_addr = desc->rio_addr; + next_addr = -1; + bcount = 0; + sys_size = dma_to_mport(dchan->device)->sys_size; + + rd_idx = ioread32(bdma_chan->regs + TSI721_DMAC_DRDCNT); + rd_idx %= (bdma_chan->bd_num + 1); + + idx = bdma_chan->wr_count_next % (bdma_chan->bd_num + 1); + if (idx == bdma_chan->bd_num) { + /* wrap around link descriptor */ + idx = 0; + add_count++; + } + + tsi_debug(DMA, ch_dev, "DMAC%d BD ring status: rdi=%d wri=%d", + bdma_chan->id, rd_idx, idx); + + for_each_sg(desc->sg, sg, desc->sg_len, i) { + + tsi_debug(DMAV, ch_dev, "DMAC%d sg%d/%d addr: 0x%llx len: %d", + bdma_chan->id, i, desc->sg_len, + (unsigned long long)sg_dma_address(sg), sg_dma_len(sg)); + + if (sg_dma_len(sg) > TSI721_BDMA_MAX_BCOUNT) { + tsi_err(ch_dev, "DMAC%d SG entry %d is too large", + bdma_chan->id, i); + err = -EINVAL; + break; + } + + /* + * If this sg entry forms contiguous block with previous one, + * try to merge it into existing DMA descriptor + */ + if (next_addr == sg_dma_address(sg) && + bcount + sg_dma_len(sg) <= TSI721_BDMA_MAX_BCOUNT) { + /* Adjust byte count of the descriptor */ + bcount += sg_dma_len(sg); + goto entry_done; + } else if (next_addr != -1) { + /* Finalize descriptor using total byte count value */ + tsi721_desc_fill_end(bd_ptr, bcount, 0); + tsi_debug(DMAV, ch_dev, "DMAC%d prev desc final len: %d", + bdma_chan->id, bcount); + } + + desc->rio_addr = rio_addr; + + if (i && idx == rd_idx) { + tsi_debug(DMAV, ch_dev, + "DMAC%d HW descriptor ring is full @ %d", + bdma_chan->id, i); + desc->sg = sg; + desc->sg_len -= i; + break; + } + + bd_ptr = &((struct tsi721_dma_desc *)bdma_chan->bd_base)[idx]; + err = tsi721_desc_fill_init(desc, bd_ptr, sg, sys_size); + if (err) { + tsi_err(ch_dev, "Failed to build desc: err=%d", err); + break; + } + + tsi_debug(DMAV, ch_dev, "DMAC%d bd_ptr = %p did=%d raddr=0x%llx", + bdma_chan->id, bd_ptr, desc->destid, desc->rio_addr); + + next_addr = sg_dma_address(sg); + bcount = sg_dma_len(sg); + + add_count++; + if (++idx == bdma_chan->bd_num) { + /* wrap around link descriptor */ + idx = 0; + add_count++; + } + +entry_done: + if (sg_is_last(sg)) { + tsi721_desc_fill_end(bd_ptr, bcount, 0); + tsi_debug(DMAV, ch_dev, + "DMAC%d last desc final len: %d", + bdma_chan->id, bcount); + desc->sg_len = 0; + } else { + rio_addr += sg_dma_len(sg); + next_addr += sg_dma_len(sg); + } + } + + if (!err) + bdma_chan->wr_count_next += add_count; + + return err; +} + +static void tsi721_advance_work(struct tsi721_bdma_chan *bdma_chan, + struct tsi721_tx_desc *desc) +{ + int err; + + tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d", bdma_chan->id); + + if (!tsi721_dma_is_idle(bdma_chan)) + return; + + /* + * If there is no data transfer in progress, fetch new descriptor from + * the pending queue. + */ + if (!desc && !bdma_chan->active_tx && !list_empty(&bdma_chan->queue)) { + desc = list_first_entry(&bdma_chan->queue, + struct tsi721_tx_desc, desc_node); + list_del_init((&desc->desc_node)); + bdma_chan->active_tx = desc; + } + + if (desc) { + err = tsi721_submit_sg(desc); + if (!err) + tsi721_start_dma(bdma_chan); + else { + tsi721_dma_tx_err(bdma_chan, desc); + tsi_debug(DMA, &bdma_chan->dchan.dev->device, + "DMAC%d ERR: tsi721_submit_sg failed with err=%d", + bdma_chan->id, err); + } + } + + tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d Exit", + bdma_chan->id); +} + +static void tsi721_dma_tasklet(unsigned long data) +{ + struct tsi721_bdma_chan *bdma_chan = (struct tsi721_bdma_chan *)data; + u32 dmac_int, dmac_sts; + + dmac_int = ioread32(bdma_chan->regs + TSI721_DMAC_INT); + tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d_INT = 0x%x", + bdma_chan->id, dmac_int); + /* Clear channel interrupts */ + iowrite32(dmac_int, bdma_chan->regs + TSI721_DMAC_INT); + + if (dmac_int & TSI721_DMAC_INT_ERR) { + int i = 10000; + struct tsi721_tx_desc *desc; + + desc = bdma_chan->active_tx; + dmac_sts = ioread32(bdma_chan->regs + TSI721_DMAC_STS); + tsi_err(&bdma_chan->dchan.dev->device, + "DMAC%d_STS = 0x%x did=%d raddr=0x%llx", + bdma_chan->id, dmac_sts, desc->destid, desc->rio_addr); + + /* Re-initialize DMA channel if possible */ + + if ((dmac_sts & TSI721_DMAC_STS_ABORT) == 0) + goto err_out; + + tsi721_clr_stat(bdma_chan); + + spin_lock(&bdma_chan->lock); + + /* Put DMA channel into init state */ + iowrite32(TSI721_DMAC_CTL_INIT, + bdma_chan->regs + TSI721_DMAC_CTL); + do { + udelay(1); + dmac_sts = ioread32(bdma_chan->regs + TSI721_DMAC_STS); + i--; + } while ((dmac_sts & TSI721_DMAC_STS_ABORT) && i); + + if (dmac_sts & TSI721_DMAC_STS_ABORT) { + tsi_err(&bdma_chan->dchan.dev->device, + "Failed to re-initiate DMAC%d", bdma_chan->id); + spin_unlock(&bdma_chan->lock); + goto err_out; + } + + /* Setup DMA descriptor pointers */ + iowrite32(((u64)bdma_chan->bd_phys >> 32), + bdma_chan->regs + TSI721_DMAC_DPTRH); + iowrite32(((u64)bdma_chan->bd_phys & TSI721_DMAC_DPTRL_MASK), + bdma_chan->regs + TSI721_DMAC_DPTRL); + + /* Setup descriptor status FIFO */ + iowrite32(((u64)bdma_chan->sts_phys >> 32), + bdma_chan->regs + TSI721_DMAC_DSBH); + iowrite32(((u64)bdma_chan->sts_phys & TSI721_DMAC_DSBL_MASK), + bdma_chan->regs + TSI721_DMAC_DSBL); + iowrite32(TSI721_DMAC_DSSZ_SIZE(bdma_chan->sts_size), + bdma_chan->regs + TSI721_DMAC_DSSZ); + + /* Clear interrupt bits */ + iowrite32(TSI721_DMAC_INT_ALL, + bdma_chan->regs + TSI721_DMAC_INT); + + ioread32(bdma_chan->regs + TSI721_DMAC_INT); + + bdma_chan->wr_count = bdma_chan->wr_count_next = 0; + bdma_chan->sts_rdptr = 0; + udelay(10); + + desc = bdma_chan->active_tx; + desc->status = DMA_ERROR; + dma_cookie_complete(&desc->txd); + list_add(&desc->desc_node, &bdma_chan->free_list); + bdma_chan->active_tx = NULL; + if (bdma_chan->active) + tsi721_advance_work(bdma_chan, NULL); + spin_unlock(&bdma_chan->lock); + } + + if (dmac_int & TSI721_DMAC_INT_STFULL) { + tsi_err(&bdma_chan->dchan.dev->device, + "DMAC%d descriptor status FIFO is full", + bdma_chan->id); + } + + if (dmac_int & (TSI721_DMAC_INT_DONE | TSI721_DMAC_INT_IOFDONE)) { + struct tsi721_tx_desc *desc; + + tsi721_clr_stat(bdma_chan); + spin_lock(&bdma_chan->lock); + desc = bdma_chan->active_tx; + + if (desc->sg_len == 0) { + dma_async_tx_callback callback = NULL; + void *param = NULL; + + desc->status = DMA_COMPLETE; + dma_cookie_complete(&desc->txd); + if (desc->txd.flags & DMA_PREP_INTERRUPT) { + callback = desc->txd.callback; + param = desc->txd.callback_param; + } + list_add(&desc->desc_node, &bdma_chan->free_list); + bdma_chan->active_tx = NULL; + if (bdma_chan->active) + tsi721_advance_work(bdma_chan, NULL); + spin_unlock(&bdma_chan->lock); + if (callback) + callback(param); + } else { + if (bdma_chan->active) + tsi721_advance_work(bdma_chan, + bdma_chan->active_tx); + spin_unlock(&bdma_chan->lock); + } + } +err_out: + /* Re-Enable BDMA channel interrupts */ + iowrite32(TSI721_DMAC_INT_ALL, bdma_chan->regs + TSI721_DMAC_INTE); +} + +static dma_cookie_t tsi721_tx_submit(struct dma_async_tx_descriptor *txd) +{ + struct tsi721_tx_desc *desc = to_tsi721_desc(txd); + struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(txd->chan); + dma_cookie_t cookie; + + /* Check if the descriptor is detached from any lists */ + if (!list_empty(&desc->desc_node)) { + tsi_err(&bdma_chan->dchan.dev->device, + "DMAC%d wrong state of descriptor %p", + bdma_chan->id, txd); + return -EIO; + } + + spin_lock_bh(&bdma_chan->lock); + + if (!bdma_chan->active) { + spin_unlock_bh(&bdma_chan->lock); + return -ENODEV; + } + + cookie = dma_cookie_assign(txd); + desc->status = DMA_IN_PROGRESS; + list_add_tail(&desc->desc_node, &bdma_chan->queue); + tsi721_advance_work(bdma_chan, NULL); + + spin_unlock_bh(&bdma_chan->lock); + return cookie; +} + +static int tsi721_alloc_chan_resources(struct dma_chan *dchan) +{ + struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); + struct tsi721_tx_desc *desc; + int i; + + tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id); + + if (bdma_chan->bd_base) + return dma_txqueue_sz; + + /* Initialize BDMA channel */ + if (tsi721_bdma_ch_init(bdma_chan, dma_desc_per_channel)) { + tsi_err(&dchan->dev->device, "Unable to initialize DMAC%d", + bdma_chan->id); + return -ENODEV; + } + + /* Allocate queue of transaction descriptors */ + desc = kcalloc(dma_txqueue_sz, sizeof(struct tsi721_tx_desc), + GFP_ATOMIC); + if (!desc) { + tsi721_bdma_ch_free(bdma_chan); + return -ENOMEM; + } + + bdma_chan->tx_desc = desc; + + for (i = 0; i < dma_txqueue_sz; i++) { + dma_async_tx_descriptor_init(&desc[i].txd, dchan); + desc[i].txd.tx_submit = tsi721_tx_submit; + desc[i].txd.flags = DMA_CTRL_ACK; + list_add(&desc[i].desc_node, &bdma_chan->free_list); + } + + dma_cookie_init(dchan); + + bdma_chan->active = true; + tsi721_bdma_interrupt_enable(bdma_chan, 1); + + return dma_txqueue_sz; +} + +static void tsi721_sync_dma_irq(struct tsi721_bdma_chan *bdma_chan) +{ + struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device); + +#ifdef CONFIG_PCI_MSI + if (priv->flags & TSI721_USING_MSIX) { + synchronize_irq(priv->msix[TSI721_VECT_DMA0_DONE + + bdma_chan->id].vector); + synchronize_irq(priv->msix[TSI721_VECT_DMA0_INT + + bdma_chan->id].vector); + } else +#endif + synchronize_irq(priv->pdev->irq); +} + +static void tsi721_free_chan_resources(struct dma_chan *dchan) +{ + struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); + + tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id); + + if (!bdma_chan->bd_base) + return; + + tsi721_bdma_interrupt_enable(bdma_chan, 0); + bdma_chan->active = false; + tsi721_sync_dma_irq(bdma_chan); + tasklet_kill(&bdma_chan->tasklet); + INIT_LIST_HEAD(&bdma_chan->free_list); + kfree(bdma_chan->tx_desc); + tsi721_bdma_ch_free(bdma_chan); +} + +static +enum dma_status tsi721_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); + enum dma_status status; + + spin_lock_bh(&bdma_chan->lock); + status = dma_cookie_status(dchan, cookie, txstate); + spin_unlock_bh(&bdma_chan->lock); + return status; +} + +static void tsi721_issue_pending(struct dma_chan *dchan) +{ + struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); + + tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id); + + spin_lock_bh(&bdma_chan->lock); + if (tsi721_dma_is_idle(bdma_chan) && bdma_chan->active) { + tsi721_advance_work(bdma_chan, NULL); + } + spin_unlock_bh(&bdma_chan->lock); +} + +static +struct dma_async_tx_descriptor *tsi721_prep_rio_sg(struct dma_chan *dchan, + struct scatterlist *sgl, unsigned int sg_len, + enum dma_transfer_direction dir, unsigned long flags, + void *tinfo) +{ + struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); + struct tsi721_tx_desc *desc; + struct rio_dma_ext *rext = tinfo; + enum dma_rtype rtype; + struct dma_async_tx_descriptor *txd = NULL; + + if (!sgl || !sg_len) { + tsi_err(&dchan->dev->device, "DMAC%d No SG list", + bdma_chan->id); + return ERR_PTR(-EINVAL); + } + + tsi_debug(DMA, &dchan->dev->device, "DMAC%d %s", bdma_chan->id, + (dir == DMA_DEV_TO_MEM)?"READ":"WRITE"); + + if (dir == DMA_DEV_TO_MEM) + rtype = NREAD; + else if (dir == DMA_MEM_TO_DEV) { + switch (rext->wr_type) { + case RDW_ALL_NWRITE: + rtype = ALL_NWRITE; + break; + case RDW_ALL_NWRITE_R: + rtype = ALL_NWRITE_R; + break; + case RDW_LAST_NWRITE_R: + default: + rtype = LAST_NWRITE_R; + break; + } + } else { + tsi_err(&dchan->dev->device, + "DMAC%d Unsupported DMA direction option", + bdma_chan->id); + return ERR_PTR(-EINVAL); + } + + spin_lock_bh(&bdma_chan->lock); + + if (!list_empty(&bdma_chan->free_list)) { + desc = list_first_entry(&bdma_chan->free_list, + struct tsi721_tx_desc, desc_node); + list_del_init(&desc->desc_node); + desc->destid = rext->destid; + desc->rio_addr = rext->rio_addr; + desc->rio_addr_u = 0; + desc->rtype = rtype; + desc->sg_len = sg_len; + desc->sg = sgl; + txd = &desc->txd; + txd->flags = flags; + } + + spin_unlock_bh(&bdma_chan->lock); + + if (!txd) { + tsi_debug(DMA, &dchan->dev->device, + "DMAC%d free TXD is not available", bdma_chan->id); + return ERR_PTR(-EBUSY); + } + + return txd; +} + +static int tsi721_terminate_all(struct dma_chan *dchan) +{ + struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); + struct tsi721_tx_desc *desc, *_d; + LIST_HEAD(list); + + tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id); + + spin_lock_bh(&bdma_chan->lock); + + bdma_chan->active = false; + + while (!tsi721_dma_is_idle(bdma_chan)) { + + udelay(5); +#if (0) + /* make sure to stop the transfer */ + iowrite32(TSI721_DMAC_CTL_SUSP, + bdma_chan->regs + TSI721_DMAC_CTL); + + /* Wait until DMA channel stops */ + do { + dmac_int = ioread32(bdma_chan->regs + TSI721_DMAC_INT); + } while ((dmac_int & TSI721_DMAC_INT_SUSP) == 0); +#endif + } + + if (bdma_chan->active_tx) + list_add(&bdma_chan->active_tx->desc_node, &list); + list_splice_init(&bdma_chan->queue, &list); + + list_for_each_entry_safe(desc, _d, &list, desc_node) + tsi721_dma_tx_err(bdma_chan, desc); + + spin_unlock_bh(&bdma_chan->lock); + + return 0; +} + +static void tsi721_dma_stop(struct tsi721_bdma_chan *bdma_chan) +{ + if (!bdma_chan->active) + return; + spin_lock_bh(&bdma_chan->lock); + if (!tsi721_dma_is_idle(bdma_chan)) { + int timeout = 100000; + + /* stop the transfer in progress */ + iowrite32(TSI721_DMAC_CTL_SUSP, + bdma_chan->regs + TSI721_DMAC_CTL); + + /* Wait until DMA channel stops */ + while (!tsi721_dma_is_idle(bdma_chan) && --timeout) + udelay(1); + } + + spin_unlock_bh(&bdma_chan->lock); +} + +void tsi721_dma_stop_all(struct tsi721_device *priv) +{ + int i; + + for (i = 0; i < TSI721_DMA_MAXCH; i++) { + if ((i != TSI721_DMACH_MAINT) && (dma_sel & (1 << i))) + tsi721_dma_stop(&priv->bdma[i]); + } +} + +int tsi721_register_dma(struct tsi721_device *priv) +{ + int i; + int nr_channels = 0; + int err; + struct rio_mport *mport = &priv->mport; + + INIT_LIST_HEAD(&mport->dma.channels); + + for (i = 0; i < TSI721_DMA_MAXCH; i++) { + struct tsi721_bdma_chan *bdma_chan = &priv->bdma[i]; + + if ((i == TSI721_DMACH_MAINT) || (dma_sel & (1 << i)) == 0) + continue; + + bdma_chan->regs = priv->regs + TSI721_DMAC_BASE(i); + + bdma_chan->dchan.device = &mport->dma; + bdma_chan->dchan.cookie = 1; + bdma_chan->dchan.chan_id = i; + bdma_chan->id = i; + bdma_chan->active = false; + + spin_lock_init(&bdma_chan->lock); + + bdma_chan->active_tx = NULL; + INIT_LIST_HEAD(&bdma_chan->queue); + INIT_LIST_HEAD(&bdma_chan->free_list); + + tasklet_init(&bdma_chan->tasklet, tsi721_dma_tasklet, + (unsigned long)bdma_chan); + list_add_tail(&bdma_chan->dchan.device_node, + &mport->dma.channels); + nr_channels++; + } + + mport->dma.chancnt = nr_channels; + dma_cap_zero(mport->dma.cap_mask); + dma_cap_set(DMA_PRIVATE, mport->dma.cap_mask); + dma_cap_set(DMA_SLAVE, mport->dma.cap_mask); + + mport->dma.dev = &priv->pdev->dev; + mport->dma.device_alloc_chan_resources = tsi721_alloc_chan_resources; + mport->dma.device_free_chan_resources = tsi721_free_chan_resources; + mport->dma.device_tx_status = tsi721_tx_status; + mport->dma.device_issue_pending = tsi721_issue_pending; + mport->dma.device_prep_slave_sg = tsi721_prep_rio_sg; + mport->dma.device_terminate_all = tsi721_terminate_all; + + err = dma_async_device_register(&mport->dma); + if (err) + tsi_err(&priv->pdev->dev, "Failed to register DMA device"); + + return err; +} + +void tsi721_unregister_dma(struct tsi721_device *priv) +{ + struct rio_mport *mport = &priv->mport; + struct dma_chan *chan, *_c; + struct tsi721_bdma_chan *bdma_chan; + + tsi721_dma_stop_all(priv); + dma_async_device_unregister(&mport->dma); + + list_for_each_entry_safe(chan, _c, &mport->dma.channels, + device_node) { + bdma_chan = to_tsi721_chan(chan); + if (bdma_chan->active) { + tsi721_bdma_interrupt_enable(bdma_chan, 0); + bdma_chan->active = false; + tsi721_sync_dma_irq(bdma_chan); + tasklet_kill(&bdma_chan->tasklet); + INIT_LIST_HEAD(&bdma_chan->free_list); + kfree(bdma_chan->tx_desc); + tsi721_bdma_ch_free(bdma_chan); + } + + list_del(&chan->device_node); + } +} |