diff options
Diffstat (limited to 'drivers/dma/mediatek')
| -rw-r--r-- | drivers/dma/mediatek/Kconfig | 38 | ||||
| -rw-r--r-- | drivers/dma/mediatek/Makefile | 4 | ||||
| -rw-r--r-- | drivers/dma/mediatek/mtk-cqdma.c | 938 | ||||
| -rw-r--r-- | drivers/dma/mediatek/mtk-hsdma.c | 1056 | ||||
| -rw-r--r-- | drivers/dma/mediatek/mtk-uart-apdma.c | 656 |
5 files changed, 2692 insertions, 0 deletions
diff --git a/drivers/dma/mediatek/Kconfig b/drivers/dma/mediatek/Kconfig new file mode 100644 index 000000000..7a46a5455 --- /dev/null +++ b/drivers/dma/mediatek/Kconfig @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config MTK_HSDMA + tristate "MediaTek High-Speed DMA controller support" + depends on ARCH_MEDIATEK || COMPILE_TEST + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for High-Speed DMA controller on MediaTek + SoCs. + + This controller provides the channels which is dedicated to + memory-to-memory transfer to offload from CPU through ring- + based descriptor management. + +config MTK_CQDMA + tristate "MediaTek Command-Queue DMA controller support" + depends on ARCH_MEDIATEK || COMPILE_TEST + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + select ASYNC_TX_ENABLE_CHANNEL_SWITCH + help + Enable support for Command-Queue DMA controller on MediaTek + SoCs. + + This controller provides the channels which is dedicated to + memory-to-memory transfer to offload from CPU. + +config MTK_UART_APDMA + tristate "MediaTek SoCs APDMA support for UART" + depends on OF && SERIAL_8250_MT6577 + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Support for the UART DMA engine found on MediaTek MTK SoCs. + When SERIAL_8250_MT6577 is enabled, and if you want to use DMA, + you can enable the config. The DMA engine can only be used + with MediaTek SoCs. diff --git a/drivers/dma/mediatek/Makefile b/drivers/dma/mediatek/Makefile new file mode 100644 index 000000000..5ba39a5ed --- /dev/null +++ b/drivers/dma/mediatek/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_MTK_UART_APDMA) += mtk-uart-apdma.o +obj-$(CONFIG_MTK_HSDMA) += mtk-hsdma.o +obj-$(CONFIG_MTK_CQDMA) += mtk-cqdma.o diff --git a/drivers/dma/mediatek/mtk-cqdma.c b/drivers/dma/mediatek/mtk-cqdma.c new file mode 100644 index 000000000..9ae92b894 --- /dev/null +++ b/drivers/dma/mediatek/mtk-cqdma.c @@ -0,0 +1,938 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2018-2019 MediaTek Inc. + +/* + * Driver for MediaTek Command-Queue DMA Controller + * + * Author: Shun-Chih Yu <shun-chih.yu@mediatek.com> + * + */ + +#include <linux/bitops.h> +#include <linux/clk.h> +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> +#include <linux/err.h> +#include <linux/iopoll.h> +#include <linux/interrupt.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/of_dma.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/refcount.h> +#include <linux/slab.h> + +#include "../virt-dma.h" + +#define MTK_CQDMA_USEC_POLL 10 +#define MTK_CQDMA_TIMEOUT_POLL 1000 +#define MTK_CQDMA_DMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) +#define MTK_CQDMA_ALIGN_SIZE 1 + +/* The default number of virtual channel */ +#define MTK_CQDMA_NR_VCHANS 32 + +/* The default number of physical channel */ +#define MTK_CQDMA_NR_PCHANS 3 + +/* Registers for underlying dma manipulation */ +#define MTK_CQDMA_INT_FLAG 0x0 +#define MTK_CQDMA_INT_EN 0x4 +#define MTK_CQDMA_EN 0x8 +#define MTK_CQDMA_RESET 0xc +#define MTK_CQDMA_FLUSH 0x14 +#define MTK_CQDMA_SRC 0x1c +#define MTK_CQDMA_DST 0x20 +#define MTK_CQDMA_LEN1 0x24 +#define MTK_CQDMA_LEN2 0x28 +#define MTK_CQDMA_SRC2 0x60 +#define MTK_CQDMA_DST2 0x64 + +/* Registers setting */ +#define MTK_CQDMA_EN_BIT BIT(0) +#define MTK_CQDMA_INT_FLAG_BIT BIT(0) +#define MTK_CQDMA_INT_EN_BIT BIT(0) +#define MTK_CQDMA_FLUSH_BIT BIT(0) + +#define MTK_CQDMA_WARM_RST_BIT BIT(0) +#define MTK_CQDMA_HARD_RST_BIT BIT(1) + +#define MTK_CQDMA_MAX_LEN GENMASK(27, 0) +#define MTK_CQDMA_ADDR_LIMIT GENMASK(31, 0) +#define MTK_CQDMA_ADDR2_SHFIT (32) + +/** + * struct mtk_cqdma_vdesc - The struct holding info describing virtual + * descriptor (CVD) + * @vd: An instance for struct virt_dma_desc + * @len: The total data size device wants to move + * @residue: The remaining data size device will move + * @dest: The destination address device wants to move to + * @src: The source address device wants to move from + * @ch: The pointer to the corresponding dma channel + * @node: The lise_head struct to build link-list for VDs + * @parent: The pointer to the parent CVD + */ +struct mtk_cqdma_vdesc { + struct virt_dma_desc vd; + size_t len; + size_t residue; + dma_addr_t dest; + dma_addr_t src; + struct dma_chan *ch; + + struct list_head node; + struct mtk_cqdma_vdesc *parent; +}; + +/** + * struct mtk_cqdma_pchan - The struct holding info describing physical + * channel (PC) + * @queue: Queue for the PDs issued to this PC + * @base: The mapped register I/O base of this PC + * @irq: The IRQ that this PC are using + * @refcnt: Track how many VCs are using this PC + * @tasklet: Tasklet for this PC + * @lock: Lock protect agaisting multiple VCs access PC + */ +struct mtk_cqdma_pchan { + struct list_head queue; + void __iomem *base; + u32 irq; + + refcount_t refcnt; + + struct tasklet_struct tasklet; + + /* lock to protect PC */ + spinlock_t lock; +}; + +/** + * struct mtk_cqdma_vchan - The struct holding info describing virtual + * channel (VC) + * @vc: An instance for struct virt_dma_chan + * @pc: The pointer to the underlying PC + * @issue_completion: The wait for all issued descriptors completited + * @issue_synchronize: Bool indicating channel synchronization starts + */ +struct mtk_cqdma_vchan { + struct virt_dma_chan vc; + struct mtk_cqdma_pchan *pc; + struct completion issue_completion; + bool issue_synchronize; +}; + +/** + * struct mtk_cqdma_device - The struct holding info describing CQDMA + * device + * @ddev: An instance for struct dma_device + * @clk: The clock that device internal is using + * @dma_requests: The number of VCs the device supports to + * @dma_channels: The number of PCs the device supports to + * @vc: The pointer to all available VCs + * @pc: The pointer to all the underlying PCs + */ +struct mtk_cqdma_device { + struct dma_device ddev; + struct clk *clk; + + u32 dma_requests; + u32 dma_channels; + struct mtk_cqdma_vchan *vc; + struct mtk_cqdma_pchan **pc; +}; + +static struct mtk_cqdma_device *to_cqdma_dev(struct dma_chan *chan) +{ + return container_of(chan->device, struct mtk_cqdma_device, ddev); +} + +static struct mtk_cqdma_vchan *to_cqdma_vchan(struct dma_chan *chan) +{ + return container_of(chan, struct mtk_cqdma_vchan, vc.chan); +} + +static struct mtk_cqdma_vdesc *to_cqdma_vdesc(struct virt_dma_desc *vd) +{ + return container_of(vd, struct mtk_cqdma_vdesc, vd); +} + +static struct device *cqdma2dev(struct mtk_cqdma_device *cqdma) +{ + return cqdma->ddev.dev; +} + +static u32 mtk_dma_read(struct mtk_cqdma_pchan *pc, u32 reg) +{ + return readl(pc->base + reg); +} + +static void mtk_dma_write(struct mtk_cqdma_pchan *pc, u32 reg, u32 val) +{ + writel_relaxed(val, pc->base + reg); +} + +static void mtk_dma_rmw(struct mtk_cqdma_pchan *pc, u32 reg, + u32 mask, u32 set) +{ + u32 val; + + val = mtk_dma_read(pc, reg); + val &= ~mask; + val |= set; + mtk_dma_write(pc, reg, val); +} + +static void mtk_dma_set(struct mtk_cqdma_pchan *pc, u32 reg, u32 val) +{ + mtk_dma_rmw(pc, reg, 0, val); +} + +static void mtk_dma_clr(struct mtk_cqdma_pchan *pc, u32 reg, u32 val) +{ + mtk_dma_rmw(pc, reg, val, 0); +} + +static void mtk_cqdma_vdesc_free(struct virt_dma_desc *vd) +{ + kfree(to_cqdma_vdesc(vd)); +} + +static int mtk_cqdma_poll_engine_done(struct mtk_cqdma_pchan *pc, bool atomic) +{ + u32 status = 0; + + if (!atomic) + return readl_poll_timeout(pc->base + MTK_CQDMA_EN, + status, + !(status & MTK_CQDMA_EN_BIT), + MTK_CQDMA_USEC_POLL, + MTK_CQDMA_TIMEOUT_POLL); + + return readl_poll_timeout_atomic(pc->base + MTK_CQDMA_EN, + status, + !(status & MTK_CQDMA_EN_BIT), + MTK_CQDMA_USEC_POLL, + MTK_CQDMA_TIMEOUT_POLL); +} + +static int mtk_cqdma_hard_reset(struct mtk_cqdma_pchan *pc) +{ + mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT); + mtk_dma_clr(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT); + + return mtk_cqdma_poll_engine_done(pc, true); +} + +static void mtk_cqdma_start(struct mtk_cqdma_pchan *pc, + struct mtk_cqdma_vdesc *cvd) +{ + /* wait for the previous transaction done */ + if (mtk_cqdma_poll_engine_done(pc, true) < 0) + dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma wait transaction timeout\n"); + + /* warm reset the dma engine for the new transaction */ + mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_WARM_RST_BIT); + if (mtk_cqdma_poll_engine_done(pc, true) < 0) + dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma warm reset timeout\n"); + + /* setup the source */ + mtk_dma_set(pc, MTK_CQDMA_SRC, cvd->src & MTK_CQDMA_ADDR_LIMIT); +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + mtk_dma_set(pc, MTK_CQDMA_SRC2, cvd->src >> MTK_CQDMA_ADDR2_SHFIT); +#else + mtk_dma_set(pc, MTK_CQDMA_SRC2, 0); +#endif + + /* setup the destination */ + mtk_dma_set(pc, MTK_CQDMA_DST, cvd->dest & MTK_CQDMA_ADDR_LIMIT); +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + mtk_dma_set(pc, MTK_CQDMA_DST2, cvd->dest >> MTK_CQDMA_ADDR2_SHFIT); +#else + mtk_dma_set(pc, MTK_CQDMA_DST2, 0); +#endif + + /* setup the length */ + mtk_dma_set(pc, MTK_CQDMA_LEN1, cvd->len); + + /* start dma engine */ + mtk_dma_set(pc, MTK_CQDMA_EN, MTK_CQDMA_EN_BIT); +} + +static void mtk_cqdma_issue_vchan_pending(struct mtk_cqdma_vchan *cvc) +{ + struct virt_dma_desc *vd, *vd2; + struct mtk_cqdma_pchan *pc = cvc->pc; + struct mtk_cqdma_vdesc *cvd; + bool trigger_engine = false; + + lockdep_assert_held(&cvc->vc.lock); + lockdep_assert_held(&pc->lock); + + list_for_each_entry_safe(vd, vd2, &cvc->vc.desc_issued, node) { + /* need to trigger dma engine if PC's queue is empty */ + if (list_empty(&pc->queue)) + trigger_engine = true; + + cvd = to_cqdma_vdesc(vd); + + /* add VD into PC's queue */ + list_add_tail(&cvd->node, &pc->queue); + + /* start the dma engine */ + if (trigger_engine) + mtk_cqdma_start(pc, cvd); + + /* remove VD from list desc_issued */ + list_del(&vd->node); + } +} + +/* + * return true if this VC is active, + * meaning that there are VDs under processing by the PC + */ +static bool mtk_cqdma_is_vchan_active(struct mtk_cqdma_vchan *cvc) +{ + struct mtk_cqdma_vdesc *cvd; + + list_for_each_entry(cvd, &cvc->pc->queue, node) + if (cvc == to_cqdma_vchan(cvd->ch)) + return true; + + return false; +} + +/* + * return the pointer of the CVD that is just consumed by the PC + */ +static struct mtk_cqdma_vdesc +*mtk_cqdma_consume_work_queue(struct mtk_cqdma_pchan *pc) +{ + struct mtk_cqdma_vchan *cvc; + struct mtk_cqdma_vdesc *cvd, *ret = NULL; + + /* consume a CVD from PC's queue */ + cvd = list_first_entry_or_null(&pc->queue, + struct mtk_cqdma_vdesc, node); + if (unlikely(!cvd || !cvd->parent)) + return NULL; + + cvc = to_cqdma_vchan(cvd->ch); + ret = cvd; + + /* update residue of the parent CVD */ + cvd->parent->residue -= cvd->len; + + /* delete CVD from PC's queue */ + list_del(&cvd->node); + + spin_lock(&cvc->vc.lock); + + /* check whether all the child CVDs completed */ + if (!cvd->parent->residue) { + /* add the parent VD into list desc_completed */ + vchan_cookie_complete(&cvd->parent->vd); + + /* setup completion if this VC is under synchronization */ + if (cvc->issue_synchronize && !mtk_cqdma_is_vchan_active(cvc)) { + complete(&cvc->issue_completion); + cvc->issue_synchronize = false; + } + } + + spin_unlock(&cvc->vc.lock); + + /* start transaction for next CVD in the queue */ + cvd = list_first_entry_or_null(&pc->queue, + struct mtk_cqdma_vdesc, node); + if (cvd) + mtk_cqdma_start(pc, cvd); + + return ret; +} + +static void mtk_cqdma_tasklet_cb(struct tasklet_struct *t) +{ + struct mtk_cqdma_pchan *pc = from_tasklet(pc, t, tasklet); + struct mtk_cqdma_vdesc *cvd = NULL; + unsigned long flags; + + spin_lock_irqsave(&pc->lock, flags); + /* consume the queue */ + cvd = mtk_cqdma_consume_work_queue(pc); + spin_unlock_irqrestore(&pc->lock, flags); + + /* submit the next CVD */ + if (cvd) { + dma_run_dependencies(&cvd->vd.tx); + + /* + * free child CVD after completion. + * the parent CVD would be freed with desc_free by user. + */ + if (cvd->parent != cvd) + kfree(cvd); + } + + /* re-enable interrupt before leaving tasklet */ + enable_irq(pc->irq); +} + +static irqreturn_t mtk_cqdma_irq(int irq, void *devid) +{ + struct mtk_cqdma_device *cqdma = devid; + irqreturn_t ret = IRQ_NONE; + bool schedule_tasklet = false; + u32 i; + + /* clear interrupt flags for each PC */ + for (i = 0; i < cqdma->dma_channels; ++i, schedule_tasklet = false) { + spin_lock(&cqdma->pc[i]->lock); + if (mtk_dma_read(cqdma->pc[i], + MTK_CQDMA_INT_FLAG) & MTK_CQDMA_INT_FLAG_BIT) { + /* clear interrupt */ + mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_FLAG, + MTK_CQDMA_INT_FLAG_BIT); + + schedule_tasklet = true; + ret = IRQ_HANDLED; + } + spin_unlock(&cqdma->pc[i]->lock); + + if (schedule_tasklet) { + /* disable interrupt */ + disable_irq_nosync(cqdma->pc[i]->irq); + + /* schedule the tasklet to handle the transactions */ + tasklet_schedule(&cqdma->pc[i]->tasklet); + } + } + + return ret; +} + +static struct virt_dma_desc *mtk_cqdma_find_active_desc(struct dma_chan *c, + dma_cookie_t cookie) +{ + struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); + struct virt_dma_desc *vd; + unsigned long flags; + + spin_lock_irqsave(&cvc->pc->lock, flags); + list_for_each_entry(vd, &cvc->pc->queue, node) + if (vd->tx.cookie == cookie) { + spin_unlock_irqrestore(&cvc->pc->lock, flags); + return vd; + } + spin_unlock_irqrestore(&cvc->pc->lock, flags); + + list_for_each_entry(vd, &cvc->vc.desc_issued, node) + if (vd->tx.cookie == cookie) + return vd; + + return NULL; +} + +static enum dma_status mtk_cqdma_tx_status(struct dma_chan *c, + dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); + struct mtk_cqdma_vdesc *cvd; + struct virt_dma_desc *vd; + enum dma_status ret; + unsigned long flags; + size_t bytes = 0; + + ret = dma_cookie_status(c, cookie, txstate); + if (ret == DMA_COMPLETE || !txstate) + return ret; + + spin_lock_irqsave(&cvc->vc.lock, flags); + vd = mtk_cqdma_find_active_desc(c, cookie); + spin_unlock_irqrestore(&cvc->vc.lock, flags); + + if (vd) { + cvd = to_cqdma_vdesc(vd); + bytes = cvd->residue; + } + + dma_set_residue(txstate, bytes); + + return ret; +} + +static void mtk_cqdma_issue_pending(struct dma_chan *c) +{ + struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); + unsigned long pc_flags; + unsigned long vc_flags; + + /* acquire PC's lock before VS's lock for lock dependency in tasklet */ + spin_lock_irqsave(&cvc->pc->lock, pc_flags); + spin_lock_irqsave(&cvc->vc.lock, vc_flags); + + if (vchan_issue_pending(&cvc->vc)) + mtk_cqdma_issue_vchan_pending(cvc); + + spin_unlock_irqrestore(&cvc->vc.lock, vc_flags); + spin_unlock_irqrestore(&cvc->pc->lock, pc_flags); +} + +static struct dma_async_tx_descriptor * +mtk_cqdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, + dma_addr_t src, size_t len, unsigned long flags) +{ + struct mtk_cqdma_vdesc **cvd; + struct dma_async_tx_descriptor *tx = NULL, *prev_tx = NULL; + size_t i, tlen, nr_vd; + + /* + * In the case that trsanction length is larger than the + * DMA engine supports, a single memcpy transaction needs + * to be separated into several DMA transactions. + * Each DMA transaction would be described by a CVD, + * and the first one is referred as the parent CVD, + * while the others are child CVDs. + * The parent CVD's tx descriptor is the only tx descriptor + * returned to the DMA user, and it should not be completed + * until all the child CVDs completed. + */ + nr_vd = DIV_ROUND_UP(len, MTK_CQDMA_MAX_LEN); + cvd = kcalloc(nr_vd, sizeof(*cvd), GFP_NOWAIT); + if (!cvd) + return NULL; + + for (i = 0; i < nr_vd; ++i) { + cvd[i] = kzalloc(sizeof(*cvd[i]), GFP_NOWAIT); + if (!cvd[i]) { + for (; i > 0; --i) + kfree(cvd[i - 1]); + return NULL; + } + + /* setup dma channel */ + cvd[i]->ch = c; + + /* setup sourece, destination, and length */ + tlen = (len > MTK_CQDMA_MAX_LEN) ? MTK_CQDMA_MAX_LEN : len; + cvd[i]->len = tlen; + cvd[i]->src = src; + cvd[i]->dest = dest; + + /* setup tx descriptor */ + tx = vchan_tx_prep(to_virt_chan(c), &cvd[i]->vd, flags); + tx->next = NULL; + + if (!i) { + cvd[0]->residue = len; + } else { + prev_tx->next = tx; + cvd[i]->residue = tlen; + } + + cvd[i]->parent = cvd[0]; + + /* update the src, dest, len, prev_tx for the next CVD */ + src += tlen; + dest += tlen; + len -= tlen; + prev_tx = tx; + } + + return &cvd[0]->vd.tx; +} + +static void mtk_cqdma_free_inactive_desc(struct dma_chan *c) +{ + struct virt_dma_chan *vc = to_virt_chan(c); + unsigned long flags; + LIST_HEAD(head); + + /* + * set desc_allocated, desc_submitted, + * and desc_issued as the candicates to be freed + */ + spin_lock_irqsave(&vc->lock, flags); + list_splice_tail_init(&vc->desc_allocated, &head); + list_splice_tail_init(&vc->desc_submitted, &head); + list_splice_tail_init(&vc->desc_issued, &head); + spin_unlock_irqrestore(&vc->lock, flags); + + /* free descriptor lists */ + vchan_dma_desc_free_list(vc, &head); +} + +static void mtk_cqdma_free_active_desc(struct dma_chan *c) +{ + struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); + bool sync_needed = false; + unsigned long pc_flags; + unsigned long vc_flags; + + /* acquire PC's lock first due to lock dependency in dma ISR */ + spin_lock_irqsave(&cvc->pc->lock, pc_flags); + spin_lock_irqsave(&cvc->vc.lock, vc_flags); + + /* synchronization is required if this VC is active */ + if (mtk_cqdma_is_vchan_active(cvc)) { + cvc->issue_synchronize = true; + sync_needed = true; + } + + spin_unlock_irqrestore(&cvc->vc.lock, vc_flags); + spin_unlock_irqrestore(&cvc->pc->lock, pc_flags); + + /* waiting for the completion of this VC */ + if (sync_needed) + wait_for_completion(&cvc->issue_completion); + + /* free all descriptors in list desc_completed */ + vchan_synchronize(&cvc->vc); + + WARN_ONCE(!list_empty(&cvc->vc.desc_completed), + "Desc pending still in list desc_completed\n"); +} + +static int mtk_cqdma_terminate_all(struct dma_chan *c) +{ + /* free descriptors not processed yet by hardware */ + mtk_cqdma_free_inactive_desc(c); + + /* free descriptors being processed by hardware */ + mtk_cqdma_free_active_desc(c); + + return 0; +} + +static int mtk_cqdma_alloc_chan_resources(struct dma_chan *c) +{ + struct mtk_cqdma_device *cqdma = to_cqdma_dev(c); + struct mtk_cqdma_vchan *vc = to_cqdma_vchan(c); + struct mtk_cqdma_pchan *pc = NULL; + u32 i, min_refcnt = U32_MAX, refcnt; + unsigned long flags; + + /* allocate PC with the minimun refcount */ + for (i = 0; i < cqdma->dma_channels; ++i) { + refcnt = refcount_read(&cqdma->pc[i]->refcnt); + if (refcnt < min_refcnt) { + pc = cqdma->pc[i]; + min_refcnt = refcnt; + } + } + + if (!pc) + return -ENOSPC; + + spin_lock_irqsave(&pc->lock, flags); + + if (!refcount_read(&pc->refcnt)) { + /* allocate PC when the refcount is zero */ + mtk_cqdma_hard_reset(pc); + + /* enable interrupt for this PC */ + mtk_dma_set(pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT); + + /* + * refcount_inc would complain increment on 0; use-after-free. + * Thus, we need to explicitly set it as 1 initially. + */ + refcount_set(&pc->refcnt, 1); + } else { + refcount_inc(&pc->refcnt); + } + + spin_unlock_irqrestore(&pc->lock, flags); + + vc->pc = pc; + + return 0; +} + +static void mtk_cqdma_free_chan_resources(struct dma_chan *c) +{ + struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); + unsigned long flags; + + /* free all descriptors in all lists on the VC */ + mtk_cqdma_terminate_all(c); + + spin_lock_irqsave(&cvc->pc->lock, flags); + + /* PC is not freed until there is no VC mapped to it */ + if (refcount_dec_and_test(&cvc->pc->refcnt)) { + /* start the flush operation and stop the engine */ + mtk_dma_set(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT); + + /* wait for the completion of flush operation */ + if (mtk_cqdma_poll_engine_done(cvc->pc, true) < 0) + dev_err(cqdma2dev(to_cqdma_dev(c)), "cqdma flush timeout\n"); + + /* clear the flush bit and interrupt flag */ + mtk_dma_clr(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT); + mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_FLAG, + MTK_CQDMA_INT_FLAG_BIT); + + /* disable interrupt for this PC */ + mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT); + } + + spin_unlock_irqrestore(&cvc->pc->lock, flags); +} + +static int mtk_cqdma_hw_init(struct mtk_cqdma_device *cqdma) +{ + unsigned long flags; + int err; + u32 i; + + pm_runtime_enable(cqdma2dev(cqdma)); + pm_runtime_get_sync(cqdma2dev(cqdma)); + + err = clk_prepare_enable(cqdma->clk); + + if (err) { + pm_runtime_put_sync(cqdma2dev(cqdma)); + pm_runtime_disable(cqdma2dev(cqdma)); + return err; + } + + /* reset all PCs */ + for (i = 0; i < cqdma->dma_channels; ++i) { + spin_lock_irqsave(&cqdma->pc[i]->lock, flags); + if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0) { + dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n"); + spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags); + + clk_disable_unprepare(cqdma->clk); + pm_runtime_put_sync(cqdma2dev(cqdma)); + pm_runtime_disable(cqdma2dev(cqdma)); + return -EINVAL; + } + spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags); + } + + return 0; +} + +static void mtk_cqdma_hw_deinit(struct mtk_cqdma_device *cqdma) +{ + unsigned long flags; + u32 i; + + /* reset all PCs */ + for (i = 0; i < cqdma->dma_channels; ++i) { + spin_lock_irqsave(&cqdma->pc[i]->lock, flags); + if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0) + dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n"); + spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags); + } + + clk_disable_unprepare(cqdma->clk); + + pm_runtime_put_sync(cqdma2dev(cqdma)); + pm_runtime_disable(cqdma2dev(cqdma)); +} + +static const struct of_device_id mtk_cqdma_match[] = { + { .compatible = "mediatek,mt6765-cqdma" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, mtk_cqdma_match); + +static int mtk_cqdma_probe(struct platform_device *pdev) +{ + struct mtk_cqdma_device *cqdma; + struct mtk_cqdma_vchan *vc; + struct dma_device *dd; + int err; + u32 i; + + cqdma = devm_kzalloc(&pdev->dev, sizeof(*cqdma), GFP_KERNEL); + if (!cqdma) + return -ENOMEM; + + dd = &cqdma->ddev; + + cqdma->clk = devm_clk_get(&pdev->dev, "cqdma"); + if (IS_ERR(cqdma->clk)) { + dev_err(&pdev->dev, "No clock for %s\n", + dev_name(&pdev->dev)); + return PTR_ERR(cqdma->clk); + } + + dma_cap_set(DMA_MEMCPY, dd->cap_mask); + + dd->copy_align = MTK_CQDMA_ALIGN_SIZE; + dd->device_alloc_chan_resources = mtk_cqdma_alloc_chan_resources; + dd->device_free_chan_resources = mtk_cqdma_free_chan_resources; + dd->device_tx_status = mtk_cqdma_tx_status; + dd->device_issue_pending = mtk_cqdma_issue_pending; + dd->device_prep_dma_memcpy = mtk_cqdma_prep_dma_memcpy; + dd->device_terminate_all = mtk_cqdma_terminate_all; + dd->src_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS; + dd->dst_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS; + dd->directions = BIT(DMA_MEM_TO_MEM); + dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; + dd->dev = &pdev->dev; + INIT_LIST_HEAD(&dd->channels); + + if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node, + "dma-requests", + &cqdma->dma_requests)) { + dev_info(&pdev->dev, + "Using %u as missing dma-requests property\n", + MTK_CQDMA_NR_VCHANS); + + cqdma->dma_requests = MTK_CQDMA_NR_VCHANS; + } + + if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node, + "dma-channels", + &cqdma->dma_channels)) { + dev_info(&pdev->dev, + "Using %u as missing dma-channels property\n", + MTK_CQDMA_NR_PCHANS); + + cqdma->dma_channels = MTK_CQDMA_NR_PCHANS; + } + + cqdma->pc = devm_kcalloc(&pdev->dev, cqdma->dma_channels, + sizeof(*cqdma->pc), GFP_KERNEL); + if (!cqdma->pc) + return -ENOMEM; + + /* initialization for PCs */ + for (i = 0; i < cqdma->dma_channels; ++i) { + cqdma->pc[i] = devm_kcalloc(&pdev->dev, 1, + sizeof(**cqdma->pc), GFP_KERNEL); + if (!cqdma->pc[i]) + return -ENOMEM; + + INIT_LIST_HEAD(&cqdma->pc[i]->queue); + spin_lock_init(&cqdma->pc[i]->lock); + refcount_set(&cqdma->pc[i]->refcnt, 0); + cqdma->pc[i]->base = devm_platform_ioremap_resource(pdev, i); + if (IS_ERR(cqdma->pc[i]->base)) + return PTR_ERR(cqdma->pc[i]->base); + + /* allocate IRQ resource */ + err = platform_get_irq(pdev, i); + if (err < 0) + return err; + cqdma->pc[i]->irq = err; + + err = devm_request_irq(&pdev->dev, cqdma->pc[i]->irq, + mtk_cqdma_irq, 0, dev_name(&pdev->dev), + cqdma); + if (err) { + dev_err(&pdev->dev, + "request_irq failed with err %d\n", err); + return -EINVAL; + } + } + + /* allocate resource for VCs */ + cqdma->vc = devm_kcalloc(&pdev->dev, cqdma->dma_requests, + sizeof(*cqdma->vc), GFP_KERNEL); + if (!cqdma->vc) + return -ENOMEM; + + for (i = 0; i < cqdma->dma_requests; i++) { + vc = &cqdma->vc[i]; + vc->vc.desc_free = mtk_cqdma_vdesc_free; + vchan_init(&vc->vc, dd); + init_completion(&vc->issue_completion); + } + + err = dma_async_device_register(dd); + if (err) + return err; + + err = of_dma_controller_register(pdev->dev.of_node, + of_dma_xlate_by_chan_id, cqdma); + if (err) { + dev_err(&pdev->dev, + "MediaTek CQDMA OF registration failed %d\n", err); + goto err_unregister; + } + + err = mtk_cqdma_hw_init(cqdma); + if (err) { + dev_err(&pdev->dev, + "MediaTek CQDMA HW initialization failed %d\n", err); + goto err_unregister; + } + + platform_set_drvdata(pdev, cqdma); + + /* initialize tasklet for each PC */ + for (i = 0; i < cqdma->dma_channels; ++i) + tasklet_setup(&cqdma->pc[i]->tasklet, mtk_cqdma_tasklet_cb); + + dev_info(&pdev->dev, "MediaTek CQDMA driver registered\n"); + + return 0; + +err_unregister: + dma_async_device_unregister(dd); + + return err; +} + +static int mtk_cqdma_remove(struct platform_device *pdev) +{ + struct mtk_cqdma_device *cqdma = platform_get_drvdata(pdev); + struct mtk_cqdma_vchan *vc; + unsigned long flags; + int i; + + /* kill VC task */ + for (i = 0; i < cqdma->dma_requests; i++) { + vc = &cqdma->vc[i]; + + list_del(&vc->vc.chan.device_node); + tasklet_kill(&vc->vc.task); + } + + /* disable interrupt */ + for (i = 0; i < cqdma->dma_channels; i++) { + spin_lock_irqsave(&cqdma->pc[i]->lock, flags); + mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_EN, + MTK_CQDMA_INT_EN_BIT); + spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags); + + /* Waits for any pending IRQ handlers to complete */ + synchronize_irq(cqdma->pc[i]->irq); + + tasklet_kill(&cqdma->pc[i]->tasklet); + } + + /* disable hardware */ + mtk_cqdma_hw_deinit(cqdma); + + dma_async_device_unregister(&cqdma->ddev); + of_dma_controller_free(pdev->dev.of_node); + + return 0; +} + +static struct platform_driver mtk_cqdma_driver = { + .probe = mtk_cqdma_probe, + .remove = mtk_cqdma_remove, + .driver = { + .name = KBUILD_MODNAME, + .of_match_table = mtk_cqdma_match, + }, +}; +module_platform_driver(mtk_cqdma_driver); + +MODULE_DESCRIPTION("MediaTek CQDMA Controller Driver"); +MODULE_AUTHOR("Shun-Chih Yu <shun-chih.yu@mediatek.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/mediatek/mtk-hsdma.c b/drivers/dma/mediatek/mtk-hsdma.c new file mode 100644 index 000000000..f7717c44b --- /dev/null +++ b/drivers/dma/mediatek/mtk-hsdma.c @@ -0,0 +1,1056 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2017-2018 MediaTek Inc. + +/* + * Driver for MediaTek High-Speed DMA Controller + * + * Author: Sean Wang <sean.wang@mediatek.com> + * + */ + +#include <linux/bitops.h> +#include <linux/clk.h> +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> +#include <linux/err.h> +#include <linux/iopoll.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/of_dma.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/refcount.h> +#include <linux/slab.h> + +#include "../virt-dma.h" + +#define MTK_HSDMA_USEC_POLL 20 +#define MTK_HSDMA_TIMEOUT_POLL 200000 +#define MTK_HSDMA_DMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) + +/* The default number of virtual channel */ +#define MTK_HSDMA_NR_VCHANS 3 + +/* Only one physical channel supported */ +#define MTK_HSDMA_NR_MAX_PCHANS 1 + +/* Macro for physical descriptor (PD) manipulation */ +/* The number of PD which must be 2 of power */ +#define MTK_DMA_SIZE 64 +#define MTK_HSDMA_NEXT_DESP_IDX(x, y) (((x) + 1) & ((y) - 1)) +#define MTK_HSDMA_LAST_DESP_IDX(x, y) (((x) - 1) & ((y) - 1)) +#define MTK_HSDMA_MAX_LEN 0x3f80 +#define MTK_HSDMA_ALIGN_SIZE 4 +#define MTK_HSDMA_PLEN_MASK 0x3fff +#define MTK_HSDMA_DESC_PLEN(x) (((x) & MTK_HSDMA_PLEN_MASK) << 16) +#define MTK_HSDMA_DESC_PLEN_GET(x) (((x) >> 16) & MTK_HSDMA_PLEN_MASK) + +/* Registers for underlying ring manipulation */ +#define MTK_HSDMA_TX_BASE 0x0 +#define MTK_HSDMA_TX_CNT 0x4 +#define MTK_HSDMA_TX_CPU 0x8 +#define MTK_HSDMA_TX_DMA 0xc +#define MTK_HSDMA_RX_BASE 0x100 +#define MTK_HSDMA_RX_CNT 0x104 +#define MTK_HSDMA_RX_CPU 0x108 +#define MTK_HSDMA_RX_DMA 0x10c + +/* Registers for global setup */ +#define MTK_HSDMA_GLO 0x204 +#define MTK_HSDMA_GLO_MULTI_DMA BIT(10) +#define MTK_HSDMA_TX_WB_DDONE BIT(6) +#define MTK_HSDMA_BURST_64BYTES (0x2 << 4) +#define MTK_HSDMA_GLO_RX_BUSY BIT(3) +#define MTK_HSDMA_GLO_RX_DMA BIT(2) +#define MTK_HSDMA_GLO_TX_BUSY BIT(1) +#define MTK_HSDMA_GLO_TX_DMA BIT(0) +#define MTK_HSDMA_GLO_DMA (MTK_HSDMA_GLO_TX_DMA | \ + MTK_HSDMA_GLO_RX_DMA) +#define MTK_HSDMA_GLO_BUSY (MTK_HSDMA_GLO_RX_BUSY | \ + MTK_HSDMA_GLO_TX_BUSY) +#define MTK_HSDMA_GLO_DEFAULT (MTK_HSDMA_GLO_TX_DMA | \ + MTK_HSDMA_GLO_RX_DMA | \ + MTK_HSDMA_TX_WB_DDONE | \ + MTK_HSDMA_BURST_64BYTES | \ + MTK_HSDMA_GLO_MULTI_DMA) + +/* Registers for reset */ +#define MTK_HSDMA_RESET 0x208 +#define MTK_HSDMA_RST_TX BIT(0) +#define MTK_HSDMA_RST_RX BIT(16) + +/* Registers for interrupt control */ +#define MTK_HSDMA_DLYINT 0x20c +#define MTK_HSDMA_RXDLY_INT_EN BIT(15) + +/* Interrupt fires when the pending number's more than the specified */ +#define MTK_HSDMA_RXMAX_PINT(x) (((x) & 0x7f) << 8) + +/* Interrupt fires when the pending time's more than the specified in 20 us */ +#define MTK_HSDMA_RXMAX_PTIME(x) ((x) & 0x7f) +#define MTK_HSDMA_DLYINT_DEFAULT (MTK_HSDMA_RXDLY_INT_EN | \ + MTK_HSDMA_RXMAX_PINT(20) | \ + MTK_HSDMA_RXMAX_PTIME(20)) +#define MTK_HSDMA_INT_STATUS 0x220 +#define MTK_HSDMA_INT_ENABLE 0x228 +#define MTK_HSDMA_INT_RXDONE BIT(16) + +enum mtk_hsdma_vdesc_flag { + MTK_HSDMA_VDESC_FINISHED = 0x01, +}; + +#define IS_MTK_HSDMA_VDESC_FINISHED(x) ((x) == MTK_HSDMA_VDESC_FINISHED) + +/** + * struct mtk_hsdma_pdesc - This is the struct holding info describing physical + * descriptor (PD) and its placement must be kept at + * 4-bytes alignment in little endian order. + * @desc1: | The control pad used to indicate hardware how to + * @desc2: | deal with the descriptor such as source and + * @desc3: | destination address and data length. The maximum + * @desc4: | data length each pdesc can handle is 0x3f80 bytes + */ +struct mtk_hsdma_pdesc { + __le32 desc1; + __le32 desc2; + __le32 desc3; + __le32 desc4; +} __packed __aligned(4); + +/** + * struct mtk_hsdma_vdesc - This is the struct holding info describing virtual + * descriptor (VD) + * @vd: An instance for struct virt_dma_desc + * @len: The total data size device wants to move + * @residue: The remaining data size device will move + * @dest: The destination address device wants to move to + * @src: The source address device wants to move from + */ +struct mtk_hsdma_vdesc { + struct virt_dma_desc vd; + size_t len; + size_t residue; + dma_addr_t dest; + dma_addr_t src; +}; + +/** + * struct mtk_hsdma_cb - This is the struct holding extra info required for RX + * ring to know what relevant VD the PD is being + * mapped to. + * @vd: Pointer to the relevant VD. + * @flag: Flag indicating what action should be taken when VD + * is completed. + */ +struct mtk_hsdma_cb { + struct virt_dma_desc *vd; + enum mtk_hsdma_vdesc_flag flag; +}; + +/** + * struct mtk_hsdma_ring - This struct holds info describing underlying ring + * space + * @txd: The descriptor TX ring which describes DMA source + * information + * @rxd: The descriptor RX ring which describes DMA + * destination information + * @cb: The extra information pointed at by RX ring + * @tphys: The physical addr of TX ring + * @rphys: The physical addr of RX ring + * @cur_tptr: Pointer to the next free descriptor used by the host + * @cur_rptr: Pointer to the last done descriptor by the device + */ +struct mtk_hsdma_ring { + struct mtk_hsdma_pdesc *txd; + struct mtk_hsdma_pdesc *rxd; + struct mtk_hsdma_cb *cb; + dma_addr_t tphys; + dma_addr_t rphys; + u16 cur_tptr; + u16 cur_rptr; +}; + +/** + * struct mtk_hsdma_pchan - This is the struct holding info describing physical + * channel (PC) + * @ring: An instance for the underlying ring + * @sz_ring: Total size allocated for the ring + * @nr_free: Total number of free rooms in the ring. It would + * be accessed and updated frequently between IRQ + * context and user context to reflect whether ring + * can accept requests from VD. + */ +struct mtk_hsdma_pchan { + struct mtk_hsdma_ring ring; + size_t sz_ring; + atomic_t nr_free; +}; + +/** + * struct mtk_hsdma_vchan - This is the struct holding info describing virtual + * channel (VC) + * @vc: An instance for struct virt_dma_chan + * @issue_completion: The wait for all issued descriptors completited + * @issue_synchronize: Bool indicating channel synchronization starts + * @desc_hw_processing: List those descriptors the hardware is processing, + * which is protected by vc.lock + */ +struct mtk_hsdma_vchan { + struct virt_dma_chan vc; + struct completion issue_completion; + bool issue_synchronize; + struct list_head desc_hw_processing; +}; + +/** + * struct mtk_hsdma_soc - This is the struct holding differences among SoCs + * @ddone: Bit mask for DDONE + * @ls0: Bit mask for LS0 + */ +struct mtk_hsdma_soc { + __le32 ddone; + __le32 ls0; +}; + +/** + * struct mtk_hsdma_device - This is the struct holding info describing HSDMA + * device + * @ddev: An instance for struct dma_device + * @base: The mapped register I/O base + * @clk: The clock that device internal is using + * @irq: The IRQ that device are using + * @dma_requests: The number of VCs the device supports to + * @vc: The pointer to all available VCs + * @pc: The pointer to the underlying PC + * @pc_refcnt: Track how many VCs are using the PC + * @lock: Lock protect agaisting multiple VCs access PC + * @soc: The pointer to area holding differences among + * vaious platform + */ +struct mtk_hsdma_device { + struct dma_device ddev; + void __iomem *base; + struct clk *clk; + u32 irq; + + u32 dma_requests; + struct mtk_hsdma_vchan *vc; + struct mtk_hsdma_pchan *pc; + refcount_t pc_refcnt; + + /* Lock used to protect against multiple VCs access PC */ + spinlock_t lock; + + const struct mtk_hsdma_soc *soc; +}; + +static struct mtk_hsdma_device *to_hsdma_dev(struct dma_chan *chan) +{ + return container_of(chan->device, struct mtk_hsdma_device, ddev); +} + +static inline struct mtk_hsdma_vchan *to_hsdma_vchan(struct dma_chan *chan) +{ + return container_of(chan, struct mtk_hsdma_vchan, vc.chan); +} + +static struct mtk_hsdma_vdesc *to_hsdma_vdesc(struct virt_dma_desc *vd) +{ + return container_of(vd, struct mtk_hsdma_vdesc, vd); +} + +static struct device *hsdma2dev(struct mtk_hsdma_device *hsdma) +{ + return hsdma->ddev.dev; +} + +static u32 mtk_dma_read(struct mtk_hsdma_device *hsdma, u32 reg) +{ + return readl(hsdma->base + reg); +} + +static void mtk_dma_write(struct mtk_hsdma_device *hsdma, u32 reg, u32 val) +{ + writel(val, hsdma->base + reg); +} + +static void mtk_dma_rmw(struct mtk_hsdma_device *hsdma, u32 reg, + u32 mask, u32 set) +{ + u32 val; + + val = mtk_dma_read(hsdma, reg); + val &= ~mask; + val |= set; + mtk_dma_write(hsdma, reg, val); +} + +static void mtk_dma_set(struct mtk_hsdma_device *hsdma, u32 reg, u32 val) +{ + mtk_dma_rmw(hsdma, reg, 0, val); +} + +static void mtk_dma_clr(struct mtk_hsdma_device *hsdma, u32 reg, u32 val) +{ + mtk_dma_rmw(hsdma, reg, val, 0); +} + +static void mtk_hsdma_vdesc_free(struct virt_dma_desc *vd) +{ + kfree(container_of(vd, struct mtk_hsdma_vdesc, vd)); +} + +static int mtk_hsdma_busy_wait(struct mtk_hsdma_device *hsdma) +{ + u32 status = 0; + + return readl_poll_timeout(hsdma->base + MTK_HSDMA_GLO, status, + !(status & MTK_HSDMA_GLO_BUSY), + MTK_HSDMA_USEC_POLL, + MTK_HSDMA_TIMEOUT_POLL); +} + +static int mtk_hsdma_alloc_pchan(struct mtk_hsdma_device *hsdma, + struct mtk_hsdma_pchan *pc) +{ + struct mtk_hsdma_ring *ring = &pc->ring; + int err; + + memset(pc, 0, sizeof(*pc)); + + /* + * Allocate ring space where [0 ... MTK_DMA_SIZE - 1] is for TX ring + * and [MTK_DMA_SIZE ... 2 * MTK_DMA_SIZE - 1] is for RX ring. + */ + pc->sz_ring = 2 * MTK_DMA_SIZE * sizeof(*ring->txd); + ring->txd = dma_alloc_coherent(hsdma2dev(hsdma), pc->sz_ring, + &ring->tphys, GFP_NOWAIT); + if (!ring->txd) + return -ENOMEM; + + ring->rxd = &ring->txd[MTK_DMA_SIZE]; + ring->rphys = ring->tphys + MTK_DMA_SIZE * sizeof(*ring->txd); + ring->cur_tptr = 0; + ring->cur_rptr = MTK_DMA_SIZE - 1; + + ring->cb = kcalloc(MTK_DMA_SIZE, sizeof(*ring->cb), GFP_NOWAIT); + if (!ring->cb) { + err = -ENOMEM; + goto err_free_dma; + } + + atomic_set(&pc->nr_free, MTK_DMA_SIZE - 1); + + /* Disable HSDMA and wait for the completion */ + mtk_dma_clr(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA); + err = mtk_hsdma_busy_wait(hsdma); + if (err) + goto err_free_cb; + + /* Reset */ + mtk_dma_set(hsdma, MTK_HSDMA_RESET, + MTK_HSDMA_RST_TX | MTK_HSDMA_RST_RX); + mtk_dma_clr(hsdma, MTK_HSDMA_RESET, + MTK_HSDMA_RST_TX | MTK_HSDMA_RST_RX); + + /* Setup HSDMA initial pointer in the ring */ + mtk_dma_write(hsdma, MTK_HSDMA_TX_BASE, ring->tphys); + mtk_dma_write(hsdma, MTK_HSDMA_TX_CNT, MTK_DMA_SIZE); + mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, ring->cur_tptr); + mtk_dma_write(hsdma, MTK_HSDMA_TX_DMA, 0); + mtk_dma_write(hsdma, MTK_HSDMA_RX_BASE, ring->rphys); + mtk_dma_write(hsdma, MTK_HSDMA_RX_CNT, MTK_DMA_SIZE); + mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, ring->cur_rptr); + mtk_dma_write(hsdma, MTK_HSDMA_RX_DMA, 0); + + /* Enable HSDMA */ + mtk_dma_set(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA); + + /* Setup delayed interrupt */ + mtk_dma_write(hsdma, MTK_HSDMA_DLYINT, MTK_HSDMA_DLYINT_DEFAULT); + + /* Enable interrupt */ + mtk_dma_set(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE); + + return 0; + +err_free_cb: + kfree(ring->cb); + +err_free_dma: + dma_free_coherent(hsdma2dev(hsdma), + pc->sz_ring, ring->txd, ring->tphys); + return err; +} + +static void mtk_hsdma_free_pchan(struct mtk_hsdma_device *hsdma, + struct mtk_hsdma_pchan *pc) +{ + struct mtk_hsdma_ring *ring = &pc->ring; + + /* Disable HSDMA and then wait for the completion */ + mtk_dma_clr(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA); + mtk_hsdma_busy_wait(hsdma); + + /* Reset pointer in the ring */ + mtk_dma_clr(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE); + mtk_dma_write(hsdma, MTK_HSDMA_TX_BASE, 0); + mtk_dma_write(hsdma, MTK_HSDMA_TX_CNT, 0); + mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, 0); + mtk_dma_write(hsdma, MTK_HSDMA_RX_BASE, 0); + mtk_dma_write(hsdma, MTK_HSDMA_RX_CNT, 0); + mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, MTK_DMA_SIZE - 1); + + kfree(ring->cb); + + dma_free_coherent(hsdma2dev(hsdma), + pc->sz_ring, ring->txd, ring->tphys); +} + +static int mtk_hsdma_issue_pending_vdesc(struct mtk_hsdma_device *hsdma, + struct mtk_hsdma_pchan *pc, + struct mtk_hsdma_vdesc *hvd) +{ + struct mtk_hsdma_ring *ring = &pc->ring; + struct mtk_hsdma_pdesc *txd, *rxd; + u16 reserved, prev, tlen, num_sgs; + unsigned long flags; + + /* Protect against PC is accessed by multiple VCs simultaneously */ + spin_lock_irqsave(&hsdma->lock, flags); + + /* + * Reserve rooms, where pc->nr_free is used to track how many free + * rooms in the ring being updated in user and IRQ context. + */ + num_sgs = DIV_ROUND_UP(hvd->len, MTK_HSDMA_MAX_LEN); + reserved = min_t(u16, num_sgs, atomic_read(&pc->nr_free)); + + if (!reserved) { + spin_unlock_irqrestore(&hsdma->lock, flags); + return -ENOSPC; + } + + atomic_sub(reserved, &pc->nr_free); + + while (reserved--) { + /* Limit size by PD capability for valid data moving */ + tlen = (hvd->len > MTK_HSDMA_MAX_LEN) ? + MTK_HSDMA_MAX_LEN : hvd->len; + + /* + * Setup PDs using the remaining VD info mapped on those + * reserved rooms. And since RXD is shared memory between the + * host and the device allocated by dma_alloc_coherent call, + * the helper macro WRITE_ONCE can ensure the data written to + * RAM would really happens. + */ + txd = &ring->txd[ring->cur_tptr]; + WRITE_ONCE(txd->desc1, hvd->src); + WRITE_ONCE(txd->desc2, + hsdma->soc->ls0 | MTK_HSDMA_DESC_PLEN(tlen)); + + rxd = &ring->rxd[ring->cur_tptr]; + WRITE_ONCE(rxd->desc1, hvd->dest); + WRITE_ONCE(rxd->desc2, MTK_HSDMA_DESC_PLEN(tlen)); + + /* Associate VD, the PD belonged to */ + ring->cb[ring->cur_tptr].vd = &hvd->vd; + + /* Move forward the pointer of TX ring */ + ring->cur_tptr = MTK_HSDMA_NEXT_DESP_IDX(ring->cur_tptr, + MTK_DMA_SIZE); + + /* Update VD with remaining data */ + hvd->src += tlen; + hvd->dest += tlen; + hvd->len -= tlen; + } + + /* + * Tagging flag for the last PD for VD will be responsible for + * completing VD. + */ + if (!hvd->len) { + prev = MTK_HSDMA_LAST_DESP_IDX(ring->cur_tptr, MTK_DMA_SIZE); + ring->cb[prev].flag = MTK_HSDMA_VDESC_FINISHED; + } + + /* Ensure all changes indeed done before we're going on */ + wmb(); + + /* + * Updating into hardware the pointer of TX ring lets HSDMA to take + * action for those pending PDs. + */ + mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, ring->cur_tptr); + + spin_unlock_irqrestore(&hsdma->lock, flags); + + return 0; +} + +static void mtk_hsdma_issue_vchan_pending(struct mtk_hsdma_device *hsdma, + struct mtk_hsdma_vchan *hvc) +{ + struct virt_dma_desc *vd, *vd2; + int err; + + lockdep_assert_held(&hvc->vc.lock); + + list_for_each_entry_safe(vd, vd2, &hvc->vc.desc_issued, node) { + struct mtk_hsdma_vdesc *hvd; + + hvd = to_hsdma_vdesc(vd); + + /* Map VD into PC and all VCs shares a single PC */ + err = mtk_hsdma_issue_pending_vdesc(hsdma, hsdma->pc, hvd); + + /* + * Move VD from desc_issued to desc_hw_processing when entire + * VD is fit into available PDs. Otherwise, the uncompleted + * VDs would stay in list desc_issued and then restart the + * processing as soon as possible once underlying ring space + * got freed. + */ + if (err == -ENOSPC || hvd->len > 0) + break; + + /* + * The extra list desc_hw_processing is used because + * hardware can't provide sufficient information allowing us + * to know what VDs are still working on the underlying ring. + * Through the additional list, it can help us to implement + * terminate_all, residue calculation and such thing needed + * to know detail descriptor status on the hardware. + */ + list_move_tail(&vd->node, &hvc->desc_hw_processing); + } +} + +static void mtk_hsdma_free_rooms_in_ring(struct mtk_hsdma_device *hsdma) +{ + struct mtk_hsdma_vchan *hvc; + struct mtk_hsdma_pdesc *rxd; + struct mtk_hsdma_vdesc *hvd; + struct mtk_hsdma_pchan *pc; + struct mtk_hsdma_cb *cb; + int i = MTK_DMA_SIZE; + __le32 desc2; + u32 status; + u16 next; + + /* Read IRQ status */ + status = mtk_dma_read(hsdma, MTK_HSDMA_INT_STATUS); + if (unlikely(!(status & MTK_HSDMA_INT_RXDONE))) + goto rx_done; + + pc = hsdma->pc; + + /* + * Using a fail-safe loop with iterations of up to MTK_DMA_SIZE to + * reclaim these finished descriptors: The most number of PDs the ISR + * can handle at one time shouldn't be more than MTK_DMA_SIZE so we + * take it as limited count instead of just using a dangerous infinite + * poll. + */ + while (i--) { + next = MTK_HSDMA_NEXT_DESP_IDX(pc->ring.cur_rptr, + MTK_DMA_SIZE); + rxd = &pc->ring.rxd[next]; + + /* + * If MTK_HSDMA_DESC_DDONE is no specified, that means data + * moving for the PD is still under going. + */ + desc2 = READ_ONCE(rxd->desc2); + if (!(desc2 & hsdma->soc->ddone)) + break; + + cb = &pc->ring.cb[next]; + if (unlikely(!cb->vd)) { + dev_err(hsdma2dev(hsdma), "cb->vd cannot be null\n"); + break; + } + + /* Update residue of VD the associated PD belonged to */ + hvd = to_hsdma_vdesc(cb->vd); + hvd->residue -= MTK_HSDMA_DESC_PLEN_GET(rxd->desc2); + + /* Complete VD until the relevant last PD is finished */ + if (IS_MTK_HSDMA_VDESC_FINISHED(cb->flag)) { + hvc = to_hsdma_vchan(cb->vd->tx.chan); + + spin_lock(&hvc->vc.lock); + + /* Remove VD from list desc_hw_processing */ + list_del(&cb->vd->node); + + /* Add VD into list desc_completed */ + vchan_cookie_complete(cb->vd); + + if (hvc->issue_synchronize && + list_empty(&hvc->desc_hw_processing)) { + complete(&hvc->issue_completion); + hvc->issue_synchronize = false; + } + spin_unlock(&hvc->vc.lock); + + cb->flag = 0; + } + + cb->vd = NULL; + + /* + * Recycle the RXD with the helper WRITE_ONCE that can ensure + * data written into RAM would really happens. + */ + WRITE_ONCE(rxd->desc1, 0); + WRITE_ONCE(rxd->desc2, 0); + pc->ring.cur_rptr = next; + + /* Release rooms */ + atomic_inc(&pc->nr_free); + } + + /* Ensure all changes indeed done before we're going on */ + wmb(); + + /* Update CPU pointer for those completed PDs */ + mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, pc->ring.cur_rptr); + + /* + * Acking the pending IRQ allows hardware no longer to keep the used + * IRQ line in certain trigger state when software has completed all + * the finished physical descriptors. + */ + if (atomic_read(&pc->nr_free) >= MTK_DMA_SIZE - 1) + mtk_dma_write(hsdma, MTK_HSDMA_INT_STATUS, status); + + /* ASAP handles pending VDs in all VCs after freeing some rooms */ + for (i = 0; i < hsdma->dma_requests; i++) { + hvc = &hsdma->vc[i]; + spin_lock(&hvc->vc.lock); + mtk_hsdma_issue_vchan_pending(hsdma, hvc); + spin_unlock(&hvc->vc.lock); + } + +rx_done: + /* All completed PDs are cleaned up, so enable interrupt again */ + mtk_dma_set(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE); +} + +static irqreturn_t mtk_hsdma_irq(int irq, void *devid) +{ + struct mtk_hsdma_device *hsdma = devid; + + /* + * Disable interrupt until all completed PDs are cleaned up in + * mtk_hsdma_free_rooms call. + */ + mtk_dma_clr(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE); + + mtk_hsdma_free_rooms_in_ring(hsdma); + + return IRQ_HANDLED; +} + +static struct virt_dma_desc *mtk_hsdma_find_active_desc(struct dma_chan *c, + dma_cookie_t cookie) +{ + struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c); + struct virt_dma_desc *vd; + + list_for_each_entry(vd, &hvc->desc_hw_processing, node) + if (vd->tx.cookie == cookie) + return vd; + + list_for_each_entry(vd, &hvc->vc.desc_issued, node) + if (vd->tx.cookie == cookie) + return vd; + + return NULL; +} + +static enum dma_status mtk_hsdma_tx_status(struct dma_chan *c, + dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c); + struct mtk_hsdma_vdesc *hvd; + struct virt_dma_desc *vd; + enum dma_status ret; + unsigned long flags; + size_t bytes = 0; + + ret = dma_cookie_status(c, cookie, txstate); + if (ret == DMA_COMPLETE || !txstate) + return ret; + + spin_lock_irqsave(&hvc->vc.lock, flags); + vd = mtk_hsdma_find_active_desc(c, cookie); + spin_unlock_irqrestore(&hvc->vc.lock, flags); + + if (vd) { + hvd = to_hsdma_vdesc(vd); + bytes = hvd->residue; + } + + dma_set_residue(txstate, bytes); + + return ret; +} + +static void mtk_hsdma_issue_pending(struct dma_chan *c) +{ + struct mtk_hsdma_device *hsdma = to_hsdma_dev(c); + struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c); + unsigned long flags; + + spin_lock_irqsave(&hvc->vc.lock, flags); + + if (vchan_issue_pending(&hvc->vc)) + mtk_hsdma_issue_vchan_pending(hsdma, hvc); + + spin_unlock_irqrestore(&hvc->vc.lock, flags); +} + +static struct dma_async_tx_descriptor * +mtk_hsdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, + dma_addr_t src, size_t len, unsigned long flags) +{ + struct mtk_hsdma_vdesc *hvd; + + hvd = kzalloc(sizeof(*hvd), GFP_NOWAIT); + if (!hvd) + return NULL; + + hvd->len = len; + hvd->residue = len; + hvd->src = src; + hvd->dest = dest; + + return vchan_tx_prep(to_virt_chan(c), &hvd->vd, flags); +} + +static int mtk_hsdma_free_inactive_desc(struct dma_chan *c) +{ + struct virt_dma_chan *vc = to_virt_chan(c); + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&vc->lock, flags); + list_splice_tail_init(&vc->desc_allocated, &head); + list_splice_tail_init(&vc->desc_submitted, &head); + list_splice_tail_init(&vc->desc_issued, &head); + spin_unlock_irqrestore(&vc->lock, flags); + + /* At the point, we don't expect users put descriptor into VC again */ + vchan_dma_desc_free_list(vc, &head); + + return 0; +} + +static void mtk_hsdma_free_active_desc(struct dma_chan *c) +{ + struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c); + bool sync_needed = false; + + /* + * Once issue_synchronize is being set, which means once the hardware + * consumes all descriptors for the channel in the ring, the + * synchronization must be notified immediately it is completed. + */ + spin_lock(&hvc->vc.lock); + if (!list_empty(&hvc->desc_hw_processing)) { + hvc->issue_synchronize = true; + sync_needed = true; + } + spin_unlock(&hvc->vc.lock); + + if (sync_needed) + wait_for_completion(&hvc->issue_completion); + /* + * At the point, we expect that all remaining descriptors in the ring + * for the channel should be all processing done. + */ + WARN_ONCE(!list_empty(&hvc->desc_hw_processing), + "Desc pending still in list desc_hw_processing\n"); + + /* Free all descriptors in list desc_completed */ + vchan_synchronize(&hvc->vc); + + WARN_ONCE(!list_empty(&hvc->vc.desc_completed), + "Desc pending still in list desc_completed\n"); +} + +static int mtk_hsdma_terminate_all(struct dma_chan *c) +{ + /* + * Free pending descriptors not processed yet by hardware that have + * previously been submitted to the channel. + */ + mtk_hsdma_free_inactive_desc(c); + + /* + * However, the DMA engine doesn't provide any way to stop these + * descriptors being processed currently by hardware. The only way is + * to just waiting until these descriptors are all processed completely + * through mtk_hsdma_free_active_desc call. + */ + mtk_hsdma_free_active_desc(c); + + return 0; +} + +static int mtk_hsdma_alloc_chan_resources(struct dma_chan *c) +{ + struct mtk_hsdma_device *hsdma = to_hsdma_dev(c); + int err; + + /* + * Since HSDMA has only one PC, the resource for PC is being allocated + * when the first VC is being created and the other VCs would run on + * the same PC. + */ + if (!refcount_read(&hsdma->pc_refcnt)) { + err = mtk_hsdma_alloc_pchan(hsdma, hsdma->pc); + if (err) + return err; + /* + * refcount_inc would complain increment on 0; use-after-free. + * Thus, we need to explicitly set it as 1 initially. + */ + refcount_set(&hsdma->pc_refcnt, 1); + } else { + refcount_inc(&hsdma->pc_refcnt); + } + + return 0; +} + +static void mtk_hsdma_free_chan_resources(struct dma_chan *c) +{ + struct mtk_hsdma_device *hsdma = to_hsdma_dev(c); + + /* Free all descriptors in all lists on the VC */ + mtk_hsdma_terminate_all(c); + + /* The resource for PC is not freed until all the VCs are destroyed */ + if (!refcount_dec_and_test(&hsdma->pc_refcnt)) + return; + + mtk_hsdma_free_pchan(hsdma, hsdma->pc); +} + +static int mtk_hsdma_hw_init(struct mtk_hsdma_device *hsdma) +{ + int err; + + pm_runtime_enable(hsdma2dev(hsdma)); + pm_runtime_get_sync(hsdma2dev(hsdma)); + + err = clk_prepare_enable(hsdma->clk); + if (err) + return err; + + mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, 0); + mtk_dma_write(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DEFAULT); + + return 0; +} + +static int mtk_hsdma_hw_deinit(struct mtk_hsdma_device *hsdma) +{ + mtk_dma_write(hsdma, MTK_HSDMA_GLO, 0); + + clk_disable_unprepare(hsdma->clk); + + pm_runtime_put_sync(hsdma2dev(hsdma)); + pm_runtime_disable(hsdma2dev(hsdma)); + + return 0; +} + +static const struct mtk_hsdma_soc mt7623_soc = { + .ddone = BIT(31), + .ls0 = BIT(30), +}; + +static const struct mtk_hsdma_soc mt7622_soc = { + .ddone = BIT(15), + .ls0 = BIT(14), +}; + +static const struct of_device_id mtk_hsdma_match[] = { + { .compatible = "mediatek,mt7623-hsdma", .data = &mt7623_soc}, + { .compatible = "mediatek,mt7622-hsdma", .data = &mt7622_soc}, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, mtk_hsdma_match); + +static int mtk_hsdma_probe(struct platform_device *pdev) +{ + struct mtk_hsdma_device *hsdma; + struct mtk_hsdma_vchan *vc; + struct dma_device *dd; + struct resource *res; + int i, err; + + hsdma = devm_kzalloc(&pdev->dev, sizeof(*hsdma), GFP_KERNEL); + if (!hsdma) + return -ENOMEM; + + dd = &hsdma->ddev; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + hsdma->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(hsdma->base)) + return PTR_ERR(hsdma->base); + + hsdma->soc = of_device_get_match_data(&pdev->dev); + if (!hsdma->soc) { + dev_err(&pdev->dev, "No device match found\n"); + return -ENODEV; + } + + hsdma->clk = devm_clk_get(&pdev->dev, "hsdma"); + if (IS_ERR(hsdma->clk)) { + dev_err(&pdev->dev, "No clock for %s\n", + dev_name(&pdev->dev)); + return PTR_ERR(hsdma->clk); + } + + err = platform_get_irq(pdev, 0); + if (err < 0) + return err; + hsdma->irq = err; + + refcount_set(&hsdma->pc_refcnt, 0); + spin_lock_init(&hsdma->lock); + + dma_cap_set(DMA_MEMCPY, dd->cap_mask); + + dd->copy_align = MTK_HSDMA_ALIGN_SIZE; + dd->device_alloc_chan_resources = mtk_hsdma_alloc_chan_resources; + dd->device_free_chan_resources = mtk_hsdma_free_chan_resources; + dd->device_tx_status = mtk_hsdma_tx_status; + dd->device_issue_pending = mtk_hsdma_issue_pending; + dd->device_prep_dma_memcpy = mtk_hsdma_prep_dma_memcpy; + dd->device_terminate_all = mtk_hsdma_terminate_all; + dd->src_addr_widths = MTK_HSDMA_DMA_BUSWIDTHS; + dd->dst_addr_widths = MTK_HSDMA_DMA_BUSWIDTHS; + dd->directions = BIT(DMA_MEM_TO_MEM); + dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; + dd->dev = &pdev->dev; + INIT_LIST_HEAD(&dd->channels); + + hsdma->dma_requests = MTK_HSDMA_NR_VCHANS; + if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node, + "dma-requests", + &hsdma->dma_requests)) { + dev_info(&pdev->dev, + "Using %u as missing dma-requests property\n", + MTK_HSDMA_NR_VCHANS); + } + + hsdma->pc = devm_kcalloc(&pdev->dev, MTK_HSDMA_NR_MAX_PCHANS, + sizeof(*hsdma->pc), GFP_KERNEL); + if (!hsdma->pc) + return -ENOMEM; + + hsdma->vc = devm_kcalloc(&pdev->dev, hsdma->dma_requests, + sizeof(*hsdma->vc), GFP_KERNEL); + if (!hsdma->vc) + return -ENOMEM; + + for (i = 0; i < hsdma->dma_requests; i++) { + vc = &hsdma->vc[i]; + vc->vc.desc_free = mtk_hsdma_vdesc_free; + vchan_init(&vc->vc, dd); + init_completion(&vc->issue_completion); + INIT_LIST_HEAD(&vc->desc_hw_processing); + } + + err = dma_async_device_register(dd); + if (err) + return err; + + err = of_dma_controller_register(pdev->dev.of_node, + of_dma_xlate_by_chan_id, hsdma); + if (err) { + dev_err(&pdev->dev, + "MediaTek HSDMA OF registration failed %d\n", err); + goto err_unregister; + } + + mtk_hsdma_hw_init(hsdma); + + err = devm_request_irq(&pdev->dev, hsdma->irq, + mtk_hsdma_irq, 0, + dev_name(&pdev->dev), hsdma); + if (err) { + dev_err(&pdev->dev, + "request_irq failed with err %d\n", err); + goto err_free; + } + + platform_set_drvdata(pdev, hsdma); + + dev_info(&pdev->dev, "MediaTek HSDMA driver registered\n"); + + return 0; + +err_free: + mtk_hsdma_hw_deinit(hsdma); + of_dma_controller_free(pdev->dev.of_node); +err_unregister: + dma_async_device_unregister(dd); + + return err; +} + +static int mtk_hsdma_remove(struct platform_device *pdev) +{ + struct mtk_hsdma_device *hsdma = platform_get_drvdata(pdev); + struct mtk_hsdma_vchan *vc; + int i; + + /* Kill VC task */ + for (i = 0; i < hsdma->dma_requests; i++) { + vc = &hsdma->vc[i]; + + list_del(&vc->vc.chan.device_node); + tasklet_kill(&vc->vc.task); + } + + /* Disable DMA interrupt */ + mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, 0); + + /* Waits for any pending IRQ handlers to complete */ + synchronize_irq(hsdma->irq); + + /* Disable hardware */ + mtk_hsdma_hw_deinit(hsdma); + + dma_async_device_unregister(&hsdma->ddev); + of_dma_controller_free(pdev->dev.of_node); + + return 0; +} + +static struct platform_driver mtk_hsdma_driver = { + .probe = mtk_hsdma_probe, + .remove = mtk_hsdma_remove, + .driver = { + .name = KBUILD_MODNAME, + .of_match_table = mtk_hsdma_match, + }, +}; +module_platform_driver(mtk_hsdma_driver); + +MODULE_DESCRIPTION("MediaTek High-Speed DMA Controller Driver"); +MODULE_AUTHOR("Sean Wang <sean.wang@mediatek.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/mediatek/mtk-uart-apdma.c b/drivers/dma/mediatek/mtk-uart-apdma.c new file mode 100644 index 000000000..0acf6a92a --- /dev/null +++ b/drivers/dma/mediatek/mtk-uart-apdma.c @@ -0,0 +1,656 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * MediaTek UART APDMA driver. + * + * Copyright (c) 2019 MediaTek Inc. + * Author: Long Cheng <long.cheng@mediatek.com> + */ + +#include <linux/clk.h> +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/iopoll.h> +#include <linux/kernel.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/of_device.h> +#include <linux/of_dma.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/slab.h> +#include <linux/spinlock.h> + +#include "../virt-dma.h" + +/* The default number of virtual channel */ +#define MTK_UART_APDMA_NR_VCHANS 8 + +#define VFF_EN_B BIT(0) +#define VFF_STOP_B BIT(0) +#define VFF_FLUSH_B BIT(0) +#define VFF_4G_EN_B BIT(0) +/* rx valid size >= vff thre */ +#define VFF_RX_INT_EN_B (BIT(0) | BIT(1)) +/* tx left size >= vff thre */ +#define VFF_TX_INT_EN_B BIT(0) +#define VFF_WARM_RST_B BIT(0) +#define VFF_RX_INT_CLR_B (BIT(0) | BIT(1)) +#define VFF_TX_INT_CLR_B 0 +#define VFF_STOP_CLR_B 0 +#define VFF_EN_CLR_B 0 +#define VFF_INT_EN_CLR_B 0 +#define VFF_4G_SUPPORT_CLR_B 0 + +/* + * interrupt trigger level for tx + * if threshold is n, no polling is required to start tx. + * otherwise need polling VFF_FLUSH. + */ +#define VFF_TX_THRE(n) (n) +/* interrupt trigger level for rx */ +#define VFF_RX_THRE(n) ((n) * 3 / 4) + +#define VFF_RING_SIZE 0xffff +/* invert this bit when wrap ring head again */ +#define VFF_RING_WRAP 0x10000 + +#define VFF_INT_FLAG 0x00 +#define VFF_INT_EN 0x04 +#define VFF_EN 0x08 +#define VFF_RST 0x0c +#define VFF_STOP 0x10 +#define VFF_FLUSH 0x14 +#define VFF_ADDR 0x1c +#define VFF_LEN 0x24 +#define VFF_THRE 0x28 +#define VFF_WPT 0x2c +#define VFF_RPT 0x30 +/* TX: the buffer size HW can read. RX: the buffer size SW can read. */ +#define VFF_VALID_SIZE 0x3c +/* TX: the buffer size SW can write. RX: the buffer size HW can write. */ +#define VFF_LEFT_SIZE 0x40 +#define VFF_DEBUG_STATUS 0x50 +#define VFF_4G_SUPPORT 0x54 + +struct mtk_uart_apdmadev { + struct dma_device ddev; + struct clk *clk; + bool support_33bits; + unsigned int dma_requests; +}; + +struct mtk_uart_apdma_desc { + struct virt_dma_desc vd; + + dma_addr_t addr; + unsigned int avail_len; +}; + +struct mtk_chan { + struct virt_dma_chan vc; + struct dma_slave_config cfg; + struct mtk_uart_apdma_desc *desc; + enum dma_transfer_direction dir; + + void __iomem *base; + unsigned int irq; + + unsigned int rx_status; +}; + +static inline struct mtk_uart_apdmadev * +to_mtk_uart_apdma_dev(struct dma_device *d) +{ + return container_of(d, struct mtk_uart_apdmadev, ddev); +} + +static inline struct mtk_chan *to_mtk_uart_apdma_chan(struct dma_chan *c) +{ + return container_of(c, struct mtk_chan, vc.chan); +} + +static inline struct mtk_uart_apdma_desc *to_mtk_uart_apdma_desc + (struct dma_async_tx_descriptor *t) +{ + return container_of(t, struct mtk_uart_apdma_desc, vd.tx); +} + +static void mtk_uart_apdma_write(struct mtk_chan *c, + unsigned int reg, unsigned int val) +{ + writel(val, c->base + reg); +} + +static unsigned int mtk_uart_apdma_read(struct mtk_chan *c, unsigned int reg) +{ + return readl(c->base + reg); +} + +static void mtk_uart_apdma_desc_free(struct virt_dma_desc *vd) +{ + kfree(container_of(vd, struct mtk_uart_apdma_desc, vd)); +} + +static void mtk_uart_apdma_start_tx(struct mtk_chan *c) +{ + struct mtk_uart_apdmadev *mtkd = + to_mtk_uart_apdma_dev(c->vc.chan.device); + struct mtk_uart_apdma_desc *d = c->desc; + unsigned int wpt, vff_sz; + + vff_sz = c->cfg.dst_port_window_size; + if (!mtk_uart_apdma_read(c, VFF_LEN)) { + mtk_uart_apdma_write(c, VFF_ADDR, d->addr); + mtk_uart_apdma_write(c, VFF_LEN, vff_sz); + mtk_uart_apdma_write(c, VFF_THRE, VFF_TX_THRE(vff_sz)); + mtk_uart_apdma_write(c, VFF_WPT, 0); + mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_TX_INT_CLR_B); + + if (mtkd->support_33bits) + mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_EN_B); + } + + mtk_uart_apdma_write(c, VFF_EN, VFF_EN_B); + if (mtk_uart_apdma_read(c, VFF_EN) != VFF_EN_B) + dev_err(c->vc.chan.device->dev, "Enable TX fail\n"); + + if (!mtk_uart_apdma_read(c, VFF_LEFT_SIZE)) { + mtk_uart_apdma_write(c, VFF_INT_EN, VFF_TX_INT_EN_B); + return; + } + + wpt = mtk_uart_apdma_read(c, VFF_WPT); + + wpt += c->desc->avail_len; + if ((wpt & VFF_RING_SIZE) == vff_sz) + wpt = (wpt & VFF_RING_WRAP) ^ VFF_RING_WRAP; + + /* Let DMA start moving data */ + mtk_uart_apdma_write(c, VFF_WPT, wpt); + + /* HW auto set to 0 when left size >= threshold */ + mtk_uart_apdma_write(c, VFF_INT_EN, VFF_TX_INT_EN_B); + if (!mtk_uart_apdma_read(c, VFF_FLUSH)) + mtk_uart_apdma_write(c, VFF_FLUSH, VFF_FLUSH_B); +} + +static void mtk_uart_apdma_start_rx(struct mtk_chan *c) +{ + struct mtk_uart_apdmadev *mtkd = + to_mtk_uart_apdma_dev(c->vc.chan.device); + struct mtk_uart_apdma_desc *d = c->desc; + unsigned int vff_sz; + + vff_sz = c->cfg.src_port_window_size; + if (!mtk_uart_apdma_read(c, VFF_LEN)) { + mtk_uart_apdma_write(c, VFF_ADDR, d->addr); + mtk_uart_apdma_write(c, VFF_LEN, vff_sz); + mtk_uart_apdma_write(c, VFF_THRE, VFF_RX_THRE(vff_sz)); + mtk_uart_apdma_write(c, VFF_RPT, 0); + mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_RX_INT_CLR_B); + + if (mtkd->support_33bits) + mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_EN_B); + } + + mtk_uart_apdma_write(c, VFF_INT_EN, VFF_RX_INT_EN_B); + mtk_uart_apdma_write(c, VFF_EN, VFF_EN_B); + if (mtk_uart_apdma_read(c, VFF_EN) != VFF_EN_B) + dev_err(c->vc.chan.device->dev, "Enable RX fail\n"); +} + +static void mtk_uart_apdma_tx_handler(struct mtk_chan *c) +{ + mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_TX_INT_CLR_B); + mtk_uart_apdma_write(c, VFF_INT_EN, VFF_INT_EN_CLR_B); + mtk_uart_apdma_write(c, VFF_EN, VFF_EN_CLR_B); +} + +static void mtk_uart_apdma_rx_handler(struct mtk_chan *c) +{ + struct mtk_uart_apdma_desc *d = c->desc; + unsigned int len, wg, rg; + int cnt; + + mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_RX_INT_CLR_B); + + if (!mtk_uart_apdma_read(c, VFF_VALID_SIZE)) + return; + + mtk_uart_apdma_write(c, VFF_EN, VFF_EN_CLR_B); + mtk_uart_apdma_write(c, VFF_INT_EN, VFF_INT_EN_CLR_B); + + len = c->cfg.src_port_window_size; + rg = mtk_uart_apdma_read(c, VFF_RPT); + wg = mtk_uart_apdma_read(c, VFF_WPT); + cnt = (wg & VFF_RING_SIZE) - (rg & VFF_RING_SIZE); + + /* + * The buffer is ring buffer. If wrap bit different, + * represents the start of the next cycle for WPT + */ + if ((rg ^ wg) & VFF_RING_WRAP) + cnt += len; + + c->rx_status = d->avail_len - cnt; + mtk_uart_apdma_write(c, VFF_RPT, wg); +} + +static void mtk_uart_apdma_chan_complete_handler(struct mtk_chan *c) +{ + struct mtk_uart_apdma_desc *d = c->desc; + + if (d) { + list_del(&d->vd.node); + vchan_cookie_complete(&d->vd); + c->desc = NULL; + } +} + +static irqreturn_t mtk_uart_apdma_irq_handler(int irq, void *dev_id) +{ + struct dma_chan *chan = (struct dma_chan *)dev_id; + struct mtk_chan *c = to_mtk_uart_apdma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&c->vc.lock, flags); + if (c->dir == DMA_DEV_TO_MEM) + mtk_uart_apdma_rx_handler(c); + else if (c->dir == DMA_MEM_TO_DEV) + mtk_uart_apdma_tx_handler(c); + mtk_uart_apdma_chan_complete_handler(c); + spin_unlock_irqrestore(&c->vc.lock, flags); + + return IRQ_HANDLED; +} + +static int mtk_uart_apdma_alloc_chan_resources(struct dma_chan *chan) +{ + struct mtk_uart_apdmadev *mtkd = to_mtk_uart_apdma_dev(chan->device); + struct mtk_chan *c = to_mtk_uart_apdma_chan(chan); + unsigned int status; + int ret; + + ret = pm_runtime_resume_and_get(mtkd->ddev.dev); + if (ret < 0) { + pm_runtime_put_noidle(chan->device->dev); + return ret; + } + + mtk_uart_apdma_write(c, VFF_ADDR, 0); + mtk_uart_apdma_write(c, VFF_THRE, 0); + mtk_uart_apdma_write(c, VFF_LEN, 0); + mtk_uart_apdma_write(c, VFF_RST, VFF_WARM_RST_B); + + ret = readx_poll_timeout(readl, c->base + VFF_EN, + status, !status, 10, 100); + if (ret) + goto err_pm; + + ret = request_irq(c->irq, mtk_uart_apdma_irq_handler, + IRQF_TRIGGER_NONE, KBUILD_MODNAME, chan); + if (ret < 0) { + dev_err(chan->device->dev, "Can't request dma IRQ\n"); + ret = -EINVAL; + goto err_pm; + } + + if (mtkd->support_33bits) + mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_SUPPORT_CLR_B); + +err_pm: + pm_runtime_put_noidle(mtkd->ddev.dev); + return ret; +} + +static void mtk_uart_apdma_free_chan_resources(struct dma_chan *chan) +{ + struct mtk_uart_apdmadev *mtkd = to_mtk_uart_apdma_dev(chan->device); + struct mtk_chan *c = to_mtk_uart_apdma_chan(chan); + + free_irq(c->irq, chan); + + tasklet_kill(&c->vc.task); + + vchan_free_chan_resources(&c->vc); + + pm_runtime_put_sync(mtkd->ddev.dev); +} + +static enum dma_status mtk_uart_apdma_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct mtk_chan *c = to_mtk_uart_apdma_chan(chan); + enum dma_status ret; + + ret = dma_cookie_status(chan, cookie, txstate); + if (!txstate) + return ret; + + dma_set_residue(txstate, c->rx_status); + + return ret; +} + +/* + * dmaengine_prep_slave_single will call the function. and sglen is 1. + * 8250 uart using one ring buffer, and deal with one sg. + */ +static struct dma_async_tx_descriptor *mtk_uart_apdma_prep_slave_sg + (struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sglen, enum dma_transfer_direction dir, + unsigned long tx_flags, void *context) +{ + struct mtk_chan *c = to_mtk_uart_apdma_chan(chan); + struct mtk_uart_apdma_desc *d; + + if (!is_slave_direction(dir) || sglen != 1) + return NULL; + + /* Now allocate and setup the descriptor */ + d = kzalloc(sizeof(*d), GFP_NOWAIT); + if (!d) + return NULL; + + d->avail_len = sg_dma_len(sgl); + d->addr = sg_dma_address(sgl); + c->dir = dir; + + return vchan_tx_prep(&c->vc, &d->vd, tx_flags); +} + +static void mtk_uart_apdma_issue_pending(struct dma_chan *chan) +{ + struct mtk_chan *c = to_mtk_uart_apdma_chan(chan); + struct virt_dma_desc *vd; + unsigned long flags; + + spin_lock_irqsave(&c->vc.lock, flags); + if (vchan_issue_pending(&c->vc) && !c->desc) { + vd = vchan_next_desc(&c->vc); + c->desc = to_mtk_uart_apdma_desc(&vd->tx); + + if (c->dir == DMA_DEV_TO_MEM) + mtk_uart_apdma_start_rx(c); + else if (c->dir == DMA_MEM_TO_DEV) + mtk_uart_apdma_start_tx(c); + } + + spin_unlock_irqrestore(&c->vc.lock, flags); +} + +static int mtk_uart_apdma_slave_config(struct dma_chan *chan, + struct dma_slave_config *config) +{ + struct mtk_chan *c = to_mtk_uart_apdma_chan(chan); + + memcpy(&c->cfg, config, sizeof(*config)); + + return 0; +} + +static int mtk_uart_apdma_terminate_all(struct dma_chan *chan) +{ + struct mtk_chan *c = to_mtk_uart_apdma_chan(chan); + unsigned long flags; + unsigned int status; + LIST_HEAD(head); + int ret; + + mtk_uart_apdma_write(c, VFF_FLUSH, VFF_FLUSH_B); + + ret = readx_poll_timeout(readl, c->base + VFF_FLUSH, + status, status != VFF_FLUSH_B, 10, 100); + if (ret) + dev_err(c->vc.chan.device->dev, "flush: fail, status=0x%x\n", + mtk_uart_apdma_read(c, VFF_DEBUG_STATUS)); + + /* + * Stop need 3 steps. + * 1. set stop to 1 + * 2. wait en to 0 + * 3. set stop as 0 + */ + mtk_uart_apdma_write(c, VFF_STOP, VFF_STOP_B); + ret = readx_poll_timeout(readl, c->base + VFF_EN, + status, !status, 10, 100); + if (ret) + dev_err(c->vc.chan.device->dev, "stop: fail, status=0x%x\n", + mtk_uart_apdma_read(c, VFF_DEBUG_STATUS)); + + mtk_uart_apdma_write(c, VFF_STOP, VFF_STOP_CLR_B); + mtk_uart_apdma_write(c, VFF_INT_EN, VFF_INT_EN_CLR_B); + + if (c->dir == DMA_DEV_TO_MEM) + mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_RX_INT_CLR_B); + else if (c->dir == DMA_MEM_TO_DEV) + mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_TX_INT_CLR_B); + + synchronize_irq(c->irq); + + spin_lock_irqsave(&c->vc.lock, flags); + vchan_get_all_descriptors(&c->vc, &head); + spin_unlock_irqrestore(&c->vc.lock, flags); + + vchan_dma_desc_free_list(&c->vc, &head); + + return 0; +} + +static int mtk_uart_apdma_device_pause(struct dma_chan *chan) +{ + struct mtk_chan *c = to_mtk_uart_apdma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&c->vc.lock, flags); + + mtk_uart_apdma_write(c, VFF_EN, VFF_EN_CLR_B); + mtk_uart_apdma_write(c, VFF_INT_EN, VFF_INT_EN_CLR_B); + + spin_unlock_irqrestore(&c->vc.lock, flags); + synchronize_irq(c->irq); + + return 0; +} + +static void mtk_uart_apdma_free(struct mtk_uart_apdmadev *mtkd) +{ + while (!list_empty(&mtkd->ddev.channels)) { + struct mtk_chan *c = list_first_entry(&mtkd->ddev.channels, + struct mtk_chan, vc.chan.device_node); + + list_del(&c->vc.chan.device_node); + tasklet_kill(&c->vc.task); + } +} + +static const struct of_device_id mtk_uart_apdma_match[] = { + { .compatible = "mediatek,mt6577-uart-dma", }, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(of, mtk_uart_apdma_match); + +static int mtk_uart_apdma_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct mtk_uart_apdmadev *mtkd; + int bit_mask = 32, rc; + struct mtk_chan *c; + unsigned int i; + + mtkd = devm_kzalloc(&pdev->dev, sizeof(*mtkd), GFP_KERNEL); + if (!mtkd) + return -ENOMEM; + + mtkd->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(mtkd->clk)) { + dev_err(&pdev->dev, "No clock specified\n"); + rc = PTR_ERR(mtkd->clk); + return rc; + } + + if (of_property_read_bool(np, "mediatek,dma-33bits")) + mtkd->support_33bits = true; + + if (mtkd->support_33bits) + bit_mask = 33; + + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(bit_mask)); + if (rc) + return rc; + + dma_cap_set(DMA_SLAVE, mtkd->ddev.cap_mask); + mtkd->ddev.device_alloc_chan_resources = + mtk_uart_apdma_alloc_chan_resources; + mtkd->ddev.device_free_chan_resources = + mtk_uart_apdma_free_chan_resources; + mtkd->ddev.device_tx_status = mtk_uart_apdma_tx_status; + mtkd->ddev.device_issue_pending = mtk_uart_apdma_issue_pending; + mtkd->ddev.device_prep_slave_sg = mtk_uart_apdma_prep_slave_sg; + mtkd->ddev.device_config = mtk_uart_apdma_slave_config; + mtkd->ddev.device_pause = mtk_uart_apdma_device_pause; + mtkd->ddev.device_terminate_all = mtk_uart_apdma_terminate_all; + mtkd->ddev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE); + mtkd->ddev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE); + mtkd->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + mtkd->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; + mtkd->ddev.dev = &pdev->dev; + INIT_LIST_HEAD(&mtkd->ddev.channels); + + mtkd->dma_requests = MTK_UART_APDMA_NR_VCHANS; + if (of_property_read_u32(np, "dma-requests", &mtkd->dma_requests)) { + dev_info(&pdev->dev, + "Using %u as missing dma-requests property\n", + MTK_UART_APDMA_NR_VCHANS); + } + + for (i = 0; i < mtkd->dma_requests; i++) { + c = devm_kzalloc(mtkd->ddev.dev, sizeof(*c), GFP_KERNEL); + if (!c) { + rc = -ENODEV; + goto err_no_dma; + } + + c->base = devm_platform_ioremap_resource(pdev, i); + if (IS_ERR(c->base)) { + rc = PTR_ERR(c->base); + goto err_no_dma; + } + c->vc.desc_free = mtk_uart_apdma_desc_free; + vchan_init(&c->vc, &mtkd->ddev); + + rc = platform_get_irq(pdev, i); + if (rc < 0) + goto err_no_dma; + c->irq = rc; + } + + pm_runtime_enable(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + + rc = dma_async_device_register(&mtkd->ddev); + if (rc) + goto rpm_disable; + + platform_set_drvdata(pdev, mtkd); + + /* Device-tree DMA controller registration */ + rc = of_dma_controller_register(np, of_dma_xlate_by_chan_id, mtkd); + if (rc) + goto dma_remove; + + return rc; + +dma_remove: + dma_async_device_unregister(&mtkd->ddev); +rpm_disable: + pm_runtime_disable(&pdev->dev); +err_no_dma: + mtk_uart_apdma_free(mtkd); + return rc; +} + +static int mtk_uart_apdma_remove(struct platform_device *pdev) +{ + struct mtk_uart_apdmadev *mtkd = platform_get_drvdata(pdev); + + of_dma_controller_free(pdev->dev.of_node); + + mtk_uart_apdma_free(mtkd); + + dma_async_device_unregister(&mtkd->ddev); + + pm_runtime_disable(&pdev->dev); + + return 0; +} + +#ifdef CONFIG_PM_SLEEP +static int mtk_uart_apdma_suspend(struct device *dev) +{ + struct mtk_uart_apdmadev *mtkd = dev_get_drvdata(dev); + + if (!pm_runtime_suspended(dev)) + clk_disable_unprepare(mtkd->clk); + + return 0; +} + +static int mtk_uart_apdma_resume(struct device *dev) +{ + int ret; + struct mtk_uart_apdmadev *mtkd = dev_get_drvdata(dev); + + if (!pm_runtime_suspended(dev)) { + ret = clk_prepare_enable(mtkd->clk); + if (ret) + return ret; + } + + return 0; +} +#endif /* CONFIG_PM_SLEEP */ + +#ifdef CONFIG_PM +static int mtk_uart_apdma_runtime_suspend(struct device *dev) +{ + struct mtk_uart_apdmadev *mtkd = dev_get_drvdata(dev); + + clk_disable_unprepare(mtkd->clk); + + return 0; +} + +static int mtk_uart_apdma_runtime_resume(struct device *dev) +{ + struct mtk_uart_apdmadev *mtkd = dev_get_drvdata(dev); + + return clk_prepare_enable(mtkd->clk); +} +#endif /* CONFIG_PM */ + +static const struct dev_pm_ops mtk_uart_apdma_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(mtk_uart_apdma_suspend, mtk_uart_apdma_resume) + SET_RUNTIME_PM_OPS(mtk_uart_apdma_runtime_suspend, + mtk_uart_apdma_runtime_resume, NULL) +}; + +static struct platform_driver mtk_uart_apdma_driver = { + .probe = mtk_uart_apdma_probe, + .remove = mtk_uart_apdma_remove, + .driver = { + .name = KBUILD_MODNAME, + .pm = &mtk_uart_apdma_pm_ops, + .of_match_table = of_match_ptr(mtk_uart_apdma_match), + }, +}; + +module_platform_driver(mtk_uart_apdma_driver); + +MODULE_DESCRIPTION("MediaTek UART APDMA Controller Driver"); +MODULE_AUTHOR("Long Cheng <long.cheng@mediatek.com>"); +MODULE_LICENSE("GPL v2"); |