summaryrefslogtreecommitdiffstats
path: root/drivers/dma/mediatek
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-27 10:05:51 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-27 10:05:51 +0000
commit5d1646d90e1f2cceb9f0828f4b28318cd0ec7744 (patch)
treea94efe259b9009378be6d90eb30d2b019d95c194 /drivers/dma/mediatek
parentInitial commit. (diff)
downloadlinux-5d1646d90e1f2cceb9f0828f4b28318cd0ec7744.tar.xz
linux-5d1646d90e1f2cceb9f0828f4b28318cd0ec7744.zip
Adding upstream version 5.10.209.upstream/5.10.209
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'drivers/dma/mediatek')
-rw-r--r--drivers/dma/mediatek/Kconfig38
-rw-r--r--drivers/dma/mediatek/Makefile4
-rw-r--r--drivers/dma/mediatek/mtk-cqdma.c942
-rw-r--r--drivers/dma/mediatek/mtk-hsdma.c1059
-rw-r--r--drivers/dma/mediatek/mtk-uart-apdma.c656
5 files changed, 2699 insertions, 0 deletions
diff --git a/drivers/dma/mediatek/Kconfig b/drivers/dma/mediatek/Kconfig
new file mode 100644
index 000000000..7a46a5455
--- /dev/null
+++ b/drivers/dma/mediatek/Kconfig
@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: GPL-2.0-only
+
+config MTK_HSDMA
+ tristate "MediaTek High-Speed DMA controller support"
+ depends on ARCH_MEDIATEK || COMPILE_TEST
+ select DMA_ENGINE
+ select DMA_VIRTUAL_CHANNELS
+ help
+ Enable support for High-Speed DMA controller on MediaTek
+ SoCs.
+
+ This controller provides the channels which is dedicated to
+ memory-to-memory transfer to offload from CPU through ring-
+ based descriptor management.
+
+config MTK_CQDMA
+ tristate "MediaTek Command-Queue DMA controller support"
+ depends on ARCH_MEDIATEK || COMPILE_TEST
+ select DMA_ENGINE
+ select DMA_VIRTUAL_CHANNELS
+ select ASYNC_TX_ENABLE_CHANNEL_SWITCH
+ help
+ Enable support for Command-Queue DMA controller on MediaTek
+ SoCs.
+
+ This controller provides the channels which is dedicated to
+ memory-to-memory transfer to offload from CPU.
+
+config MTK_UART_APDMA
+ tristate "MediaTek SoCs APDMA support for UART"
+ depends on OF && SERIAL_8250_MT6577
+ select DMA_ENGINE
+ select DMA_VIRTUAL_CHANNELS
+ help
+ Support for the UART DMA engine found on MediaTek MTK SoCs.
+ When SERIAL_8250_MT6577 is enabled, and if you want to use DMA,
+ you can enable the config. The DMA engine can only be used
+ with MediaTek SoCs.
diff --git a/drivers/dma/mediatek/Makefile b/drivers/dma/mediatek/Makefile
new file mode 100644
index 000000000..5ba39a5ed
--- /dev/null
+++ b/drivers/dma/mediatek/Makefile
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0-only
+obj-$(CONFIG_MTK_UART_APDMA) += mtk-uart-apdma.o
+obj-$(CONFIG_MTK_HSDMA) += mtk-hsdma.o
+obj-$(CONFIG_MTK_CQDMA) += mtk-cqdma.o
diff --git a/drivers/dma/mediatek/mtk-cqdma.c b/drivers/dma/mediatek/mtk-cqdma.c
new file mode 100644
index 000000000..41ef9f15d
--- /dev/null
+++ b/drivers/dma/mediatek/mtk-cqdma.c
@@ -0,0 +1,942 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2018-2019 MediaTek Inc.
+
+/*
+ * Driver for MediaTek Command-Queue DMA Controller
+ *
+ * Author: Shun-Chih Yu <shun-chih.yu@mediatek.com>
+ *
+ */
+
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/iopoll.h>
+#include <linux/interrupt.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/of_dma.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/refcount.h>
+#include <linux/slab.h>
+
+#include "../virt-dma.h"
+
+#define MTK_CQDMA_USEC_POLL 10
+#define MTK_CQDMA_TIMEOUT_POLL 1000
+#define MTK_CQDMA_DMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
+#define MTK_CQDMA_ALIGN_SIZE 1
+
+/* The default number of virtual channel */
+#define MTK_CQDMA_NR_VCHANS 32
+
+/* The default number of physical channel */
+#define MTK_CQDMA_NR_PCHANS 3
+
+/* Registers for underlying dma manipulation */
+#define MTK_CQDMA_INT_FLAG 0x0
+#define MTK_CQDMA_INT_EN 0x4
+#define MTK_CQDMA_EN 0x8
+#define MTK_CQDMA_RESET 0xc
+#define MTK_CQDMA_FLUSH 0x14
+#define MTK_CQDMA_SRC 0x1c
+#define MTK_CQDMA_DST 0x20
+#define MTK_CQDMA_LEN1 0x24
+#define MTK_CQDMA_LEN2 0x28
+#define MTK_CQDMA_SRC2 0x60
+#define MTK_CQDMA_DST2 0x64
+
+/* Registers setting */
+#define MTK_CQDMA_EN_BIT BIT(0)
+#define MTK_CQDMA_INT_FLAG_BIT BIT(0)
+#define MTK_CQDMA_INT_EN_BIT BIT(0)
+#define MTK_CQDMA_FLUSH_BIT BIT(0)
+
+#define MTK_CQDMA_WARM_RST_BIT BIT(0)
+#define MTK_CQDMA_HARD_RST_BIT BIT(1)
+
+#define MTK_CQDMA_MAX_LEN GENMASK(27, 0)
+#define MTK_CQDMA_ADDR_LIMIT GENMASK(31, 0)
+#define MTK_CQDMA_ADDR2_SHFIT (32)
+
+/**
+ * struct mtk_cqdma_vdesc - The struct holding info describing virtual
+ * descriptor (CVD)
+ * @vd: An instance for struct virt_dma_desc
+ * @len: The total data size device wants to move
+ * @residue: The remaining data size device will move
+ * @dest: The destination address device wants to move to
+ * @src: The source address device wants to move from
+ * @ch: The pointer to the corresponding dma channel
+ * @node: The lise_head struct to build link-list for VDs
+ * @parent: The pointer to the parent CVD
+ */
+struct mtk_cqdma_vdesc {
+ struct virt_dma_desc vd;
+ size_t len;
+ size_t residue;
+ dma_addr_t dest;
+ dma_addr_t src;
+ struct dma_chan *ch;
+
+ struct list_head node;
+ struct mtk_cqdma_vdesc *parent;
+};
+
+/**
+ * struct mtk_cqdma_pchan - The struct holding info describing physical
+ * channel (PC)
+ * @queue: Queue for the PDs issued to this PC
+ * @base: The mapped register I/O base of this PC
+ * @irq: The IRQ that this PC are using
+ * @refcnt: Track how many VCs are using this PC
+ * @tasklet: Tasklet for this PC
+ * @lock: Lock protect agaisting multiple VCs access PC
+ */
+struct mtk_cqdma_pchan {
+ struct list_head queue;
+ void __iomem *base;
+ u32 irq;
+
+ refcount_t refcnt;
+
+ struct tasklet_struct tasklet;
+
+ /* lock to protect PC */
+ spinlock_t lock;
+};
+
+/**
+ * struct mtk_cqdma_vchan - The struct holding info describing virtual
+ * channel (VC)
+ * @vc: An instance for struct virt_dma_chan
+ * @pc: The pointer to the underlying PC
+ * @issue_completion: The wait for all issued descriptors completited
+ * @issue_synchronize: Bool indicating channel synchronization starts
+ */
+struct mtk_cqdma_vchan {
+ struct virt_dma_chan vc;
+ struct mtk_cqdma_pchan *pc;
+ struct completion issue_completion;
+ bool issue_synchronize;
+};
+
+/**
+ * struct mtk_cqdma_device - The struct holding info describing CQDMA
+ * device
+ * @ddev: An instance for struct dma_device
+ * @clk: The clock that device internal is using
+ * @dma_requests: The number of VCs the device supports to
+ * @dma_channels: The number of PCs the device supports to
+ * @vc: The pointer to all available VCs
+ * @pc: The pointer to all the underlying PCs
+ */
+struct mtk_cqdma_device {
+ struct dma_device ddev;
+ struct clk *clk;
+
+ u32 dma_requests;
+ u32 dma_channels;
+ struct mtk_cqdma_vchan *vc;
+ struct mtk_cqdma_pchan **pc;
+};
+
+static struct mtk_cqdma_device *to_cqdma_dev(struct dma_chan *chan)
+{
+ return container_of(chan->device, struct mtk_cqdma_device, ddev);
+}
+
+static struct mtk_cqdma_vchan *to_cqdma_vchan(struct dma_chan *chan)
+{
+ return container_of(chan, struct mtk_cqdma_vchan, vc.chan);
+}
+
+static struct mtk_cqdma_vdesc *to_cqdma_vdesc(struct virt_dma_desc *vd)
+{
+ return container_of(vd, struct mtk_cqdma_vdesc, vd);
+}
+
+static struct device *cqdma2dev(struct mtk_cqdma_device *cqdma)
+{
+ return cqdma->ddev.dev;
+}
+
+static u32 mtk_dma_read(struct mtk_cqdma_pchan *pc, u32 reg)
+{
+ return readl(pc->base + reg);
+}
+
+static void mtk_dma_write(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
+{
+ writel_relaxed(val, pc->base + reg);
+}
+
+static void mtk_dma_rmw(struct mtk_cqdma_pchan *pc, u32 reg,
+ u32 mask, u32 set)
+{
+ u32 val;
+
+ val = mtk_dma_read(pc, reg);
+ val &= ~mask;
+ val |= set;
+ mtk_dma_write(pc, reg, val);
+}
+
+static void mtk_dma_set(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
+{
+ mtk_dma_rmw(pc, reg, 0, val);
+}
+
+static void mtk_dma_clr(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
+{
+ mtk_dma_rmw(pc, reg, val, 0);
+}
+
+static void mtk_cqdma_vdesc_free(struct virt_dma_desc *vd)
+{
+ kfree(to_cqdma_vdesc(vd));
+}
+
+static int mtk_cqdma_poll_engine_done(struct mtk_cqdma_pchan *pc, bool atomic)
+{
+ u32 status = 0;
+
+ if (!atomic)
+ return readl_poll_timeout(pc->base + MTK_CQDMA_EN,
+ status,
+ !(status & MTK_CQDMA_EN_BIT),
+ MTK_CQDMA_USEC_POLL,
+ MTK_CQDMA_TIMEOUT_POLL);
+
+ return readl_poll_timeout_atomic(pc->base + MTK_CQDMA_EN,
+ status,
+ !(status & MTK_CQDMA_EN_BIT),
+ MTK_CQDMA_USEC_POLL,
+ MTK_CQDMA_TIMEOUT_POLL);
+}
+
+static int mtk_cqdma_hard_reset(struct mtk_cqdma_pchan *pc)
+{
+ mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);
+ mtk_dma_clr(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);
+
+ return mtk_cqdma_poll_engine_done(pc, true);
+}
+
+static void mtk_cqdma_start(struct mtk_cqdma_pchan *pc,
+ struct mtk_cqdma_vdesc *cvd)
+{
+ /* wait for the previous transaction done */
+ if (mtk_cqdma_poll_engine_done(pc, true) < 0)
+ dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma wait transaction timeout\n");
+
+ /* warm reset the dma engine for the new transaction */
+ mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_WARM_RST_BIT);
+ if (mtk_cqdma_poll_engine_done(pc, true) < 0)
+ dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma warm reset timeout\n");
+
+ /* setup the source */
+ mtk_dma_set(pc, MTK_CQDMA_SRC, cvd->src & MTK_CQDMA_ADDR_LIMIT);
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+ mtk_dma_set(pc, MTK_CQDMA_SRC2, cvd->src >> MTK_CQDMA_ADDR2_SHFIT);
+#else
+ mtk_dma_set(pc, MTK_CQDMA_SRC2, 0);
+#endif
+
+ /* setup the destination */
+ mtk_dma_set(pc, MTK_CQDMA_DST, cvd->dest & MTK_CQDMA_ADDR_LIMIT);
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+ mtk_dma_set(pc, MTK_CQDMA_DST2, cvd->dest >> MTK_CQDMA_ADDR2_SHFIT);
+#else
+ mtk_dma_set(pc, MTK_CQDMA_DST2, 0);
+#endif
+
+ /* setup the length */
+ mtk_dma_set(pc, MTK_CQDMA_LEN1, cvd->len);
+
+ /* start dma engine */
+ mtk_dma_set(pc, MTK_CQDMA_EN, MTK_CQDMA_EN_BIT);
+}
+
+static void mtk_cqdma_issue_vchan_pending(struct mtk_cqdma_vchan *cvc)
+{
+ struct virt_dma_desc *vd, *vd2;
+ struct mtk_cqdma_pchan *pc = cvc->pc;
+ struct mtk_cqdma_vdesc *cvd;
+ bool trigger_engine = false;
+
+ lockdep_assert_held(&cvc->vc.lock);
+ lockdep_assert_held(&pc->lock);
+
+ list_for_each_entry_safe(vd, vd2, &cvc->vc.desc_issued, node) {
+ /* need to trigger dma engine if PC's queue is empty */
+ if (list_empty(&pc->queue))
+ trigger_engine = true;
+
+ cvd = to_cqdma_vdesc(vd);
+
+ /* add VD into PC's queue */
+ list_add_tail(&cvd->node, &pc->queue);
+
+ /* start the dma engine */
+ if (trigger_engine)
+ mtk_cqdma_start(pc, cvd);
+
+ /* remove VD from list desc_issued */
+ list_del(&vd->node);
+ }
+}
+
+/*
+ * return true if this VC is active,
+ * meaning that there are VDs under processing by the PC
+ */
+static bool mtk_cqdma_is_vchan_active(struct mtk_cqdma_vchan *cvc)
+{
+ struct mtk_cqdma_vdesc *cvd;
+
+ list_for_each_entry(cvd, &cvc->pc->queue, node)
+ if (cvc == to_cqdma_vchan(cvd->ch))
+ return true;
+
+ return false;
+}
+
+/*
+ * return the pointer of the CVD that is just consumed by the PC
+ */
+static struct mtk_cqdma_vdesc
+*mtk_cqdma_consume_work_queue(struct mtk_cqdma_pchan *pc)
+{
+ struct mtk_cqdma_vchan *cvc;
+ struct mtk_cqdma_vdesc *cvd, *ret = NULL;
+
+ /* consume a CVD from PC's queue */
+ cvd = list_first_entry_or_null(&pc->queue,
+ struct mtk_cqdma_vdesc, node);
+ if (unlikely(!cvd || !cvd->parent))
+ return NULL;
+
+ cvc = to_cqdma_vchan(cvd->ch);
+ ret = cvd;
+
+ /* update residue of the parent CVD */
+ cvd->parent->residue -= cvd->len;
+
+ /* delete CVD from PC's queue */
+ list_del(&cvd->node);
+
+ spin_lock(&cvc->vc.lock);
+
+ /* check whether all the child CVDs completed */
+ if (!cvd->parent->residue) {
+ /* add the parent VD into list desc_completed */
+ vchan_cookie_complete(&cvd->parent->vd);
+
+ /* setup completion if this VC is under synchronization */
+ if (cvc->issue_synchronize && !mtk_cqdma_is_vchan_active(cvc)) {
+ complete(&cvc->issue_completion);
+ cvc->issue_synchronize = false;
+ }
+ }
+
+ spin_unlock(&cvc->vc.lock);
+
+ /* start transaction for next CVD in the queue */
+ cvd = list_first_entry_or_null(&pc->queue,
+ struct mtk_cqdma_vdesc, node);
+ if (cvd)
+ mtk_cqdma_start(pc, cvd);
+
+ return ret;
+}
+
+static void mtk_cqdma_tasklet_cb(struct tasklet_struct *t)
+{
+ struct mtk_cqdma_pchan *pc = from_tasklet(pc, t, tasklet);
+ struct mtk_cqdma_vdesc *cvd = NULL;
+ unsigned long flags;
+
+ spin_lock_irqsave(&pc->lock, flags);
+ /* consume the queue */
+ cvd = mtk_cqdma_consume_work_queue(pc);
+ spin_unlock_irqrestore(&pc->lock, flags);
+
+ /* submit the next CVD */
+ if (cvd) {
+ dma_run_dependencies(&cvd->vd.tx);
+
+ /*
+ * free child CVD after completion.
+ * the parent CVD would be freeed with desc_free by user.
+ */
+ if (cvd->parent != cvd)
+ kfree(cvd);
+ }
+
+ /* re-enable interrupt before leaving tasklet */
+ enable_irq(pc->irq);
+}
+
+static irqreturn_t mtk_cqdma_irq(int irq, void *devid)
+{
+ struct mtk_cqdma_device *cqdma = devid;
+ irqreturn_t ret = IRQ_NONE;
+ bool schedule_tasklet = false;
+ u32 i;
+
+ /* clear interrupt flags for each PC */
+ for (i = 0; i < cqdma->dma_channels; ++i, schedule_tasklet = false) {
+ spin_lock(&cqdma->pc[i]->lock);
+ if (mtk_dma_read(cqdma->pc[i],
+ MTK_CQDMA_INT_FLAG) & MTK_CQDMA_INT_FLAG_BIT) {
+ /* clear interrupt */
+ mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_FLAG,
+ MTK_CQDMA_INT_FLAG_BIT);
+
+ schedule_tasklet = true;
+ ret = IRQ_HANDLED;
+ }
+ spin_unlock(&cqdma->pc[i]->lock);
+
+ if (schedule_tasklet) {
+ /* disable interrupt */
+ disable_irq_nosync(cqdma->pc[i]->irq);
+
+ /* schedule the tasklet to handle the transactions */
+ tasklet_schedule(&cqdma->pc[i]->tasklet);
+ }
+ }
+
+ return ret;
+}
+
+static struct virt_dma_desc *mtk_cqdma_find_active_desc(struct dma_chan *c,
+ dma_cookie_t cookie)
+{
+ struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
+ struct virt_dma_desc *vd;
+ unsigned long flags;
+
+ spin_lock_irqsave(&cvc->pc->lock, flags);
+ list_for_each_entry(vd, &cvc->pc->queue, node)
+ if (vd->tx.cookie == cookie) {
+ spin_unlock_irqrestore(&cvc->pc->lock, flags);
+ return vd;
+ }
+ spin_unlock_irqrestore(&cvc->pc->lock, flags);
+
+ list_for_each_entry(vd, &cvc->vc.desc_issued, node)
+ if (vd->tx.cookie == cookie)
+ return vd;
+
+ return NULL;
+}
+
+static enum dma_status mtk_cqdma_tx_status(struct dma_chan *c,
+ dma_cookie_t cookie,
+ struct dma_tx_state *txstate)
+{
+ struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
+ struct mtk_cqdma_vdesc *cvd;
+ struct virt_dma_desc *vd;
+ enum dma_status ret;
+ unsigned long flags;
+ size_t bytes = 0;
+
+ ret = dma_cookie_status(c, cookie, txstate);
+ if (ret == DMA_COMPLETE || !txstate)
+ return ret;
+
+ spin_lock_irqsave(&cvc->vc.lock, flags);
+ vd = mtk_cqdma_find_active_desc(c, cookie);
+ spin_unlock_irqrestore(&cvc->vc.lock, flags);
+
+ if (vd) {
+ cvd = to_cqdma_vdesc(vd);
+ bytes = cvd->residue;
+ }
+
+ dma_set_residue(txstate, bytes);
+
+ return ret;
+}
+
+static void mtk_cqdma_issue_pending(struct dma_chan *c)
+{
+ struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
+ unsigned long pc_flags;
+ unsigned long vc_flags;
+
+ /* acquire PC's lock before VS's lock for lock dependency in tasklet */
+ spin_lock_irqsave(&cvc->pc->lock, pc_flags);
+ spin_lock_irqsave(&cvc->vc.lock, vc_flags);
+
+ if (vchan_issue_pending(&cvc->vc))
+ mtk_cqdma_issue_vchan_pending(cvc);
+
+ spin_unlock_irqrestore(&cvc->vc.lock, vc_flags);
+ spin_unlock_irqrestore(&cvc->pc->lock, pc_flags);
+}
+
+static struct dma_async_tx_descriptor *
+mtk_cqdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest,
+ dma_addr_t src, size_t len, unsigned long flags)
+{
+ struct mtk_cqdma_vdesc **cvd;
+ struct dma_async_tx_descriptor *tx = NULL, *prev_tx = NULL;
+ size_t i, tlen, nr_vd;
+
+ /*
+ * In the case that trsanction length is larger than the
+ * DMA engine supports, a single memcpy transaction needs
+ * to be separated into several DMA transactions.
+ * Each DMA transaction would be described by a CVD,
+ * and the first one is referred as the parent CVD,
+ * while the others are child CVDs.
+ * The parent CVD's tx descriptor is the only tx descriptor
+ * returned to the DMA user, and it should not be completed
+ * until all the child CVDs completed.
+ */
+ nr_vd = DIV_ROUND_UP(len, MTK_CQDMA_MAX_LEN);
+ cvd = kcalloc(nr_vd, sizeof(*cvd), GFP_NOWAIT);
+ if (!cvd)
+ return NULL;
+
+ for (i = 0; i < nr_vd; ++i) {
+ cvd[i] = kzalloc(sizeof(*cvd[i]), GFP_NOWAIT);
+ if (!cvd[i]) {
+ for (; i > 0; --i)
+ kfree(cvd[i - 1]);
+ return NULL;
+ }
+
+ /* setup dma channel */
+ cvd[i]->ch = c;
+
+ /* setup sourece, destination, and length */
+ tlen = (len > MTK_CQDMA_MAX_LEN) ? MTK_CQDMA_MAX_LEN : len;
+ cvd[i]->len = tlen;
+ cvd[i]->src = src;
+ cvd[i]->dest = dest;
+
+ /* setup tx descriptor */
+ tx = vchan_tx_prep(to_virt_chan(c), &cvd[i]->vd, flags);
+ tx->next = NULL;
+
+ if (!i) {
+ cvd[0]->residue = len;
+ } else {
+ prev_tx->next = tx;
+ cvd[i]->residue = tlen;
+ }
+
+ cvd[i]->parent = cvd[0];
+
+ /* update the src, dest, len, prev_tx for the next CVD */
+ src += tlen;
+ dest += tlen;
+ len -= tlen;
+ prev_tx = tx;
+ }
+
+ return &cvd[0]->vd.tx;
+}
+
+static void mtk_cqdma_free_inactive_desc(struct dma_chan *c)
+{
+ struct virt_dma_chan *vc = to_virt_chan(c);
+ unsigned long flags;
+ LIST_HEAD(head);
+
+ /*
+ * set desc_allocated, desc_submitted,
+ * and desc_issued as the candicates to be freed
+ */
+ spin_lock_irqsave(&vc->lock, flags);
+ list_splice_tail_init(&vc->desc_allocated, &head);
+ list_splice_tail_init(&vc->desc_submitted, &head);
+ list_splice_tail_init(&vc->desc_issued, &head);
+ spin_unlock_irqrestore(&vc->lock, flags);
+
+ /* free descriptor lists */
+ vchan_dma_desc_free_list(vc, &head);
+}
+
+static void mtk_cqdma_free_active_desc(struct dma_chan *c)
+{
+ struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
+ bool sync_needed = false;
+ unsigned long pc_flags;
+ unsigned long vc_flags;
+
+ /* acquire PC's lock first due to lock dependency in dma ISR */
+ spin_lock_irqsave(&cvc->pc->lock, pc_flags);
+ spin_lock_irqsave(&cvc->vc.lock, vc_flags);
+
+ /* synchronization is required if this VC is active */
+ if (mtk_cqdma_is_vchan_active(cvc)) {
+ cvc->issue_synchronize = true;
+ sync_needed = true;
+ }
+
+ spin_unlock_irqrestore(&cvc->vc.lock, vc_flags);
+ spin_unlock_irqrestore(&cvc->pc->lock, pc_flags);
+
+ /* waiting for the completion of this VC */
+ if (sync_needed)
+ wait_for_completion(&cvc->issue_completion);
+
+ /* free all descriptors in list desc_completed */
+ vchan_synchronize(&cvc->vc);
+
+ WARN_ONCE(!list_empty(&cvc->vc.desc_completed),
+ "Desc pending still in list desc_completed\n");
+}
+
+static int mtk_cqdma_terminate_all(struct dma_chan *c)
+{
+ /* free descriptors not processed yet by hardware */
+ mtk_cqdma_free_inactive_desc(c);
+
+ /* free descriptors being processed by hardware */
+ mtk_cqdma_free_active_desc(c);
+
+ return 0;
+}
+
+static int mtk_cqdma_alloc_chan_resources(struct dma_chan *c)
+{
+ struct mtk_cqdma_device *cqdma = to_cqdma_dev(c);
+ struct mtk_cqdma_vchan *vc = to_cqdma_vchan(c);
+ struct mtk_cqdma_pchan *pc = NULL;
+ u32 i, min_refcnt = U32_MAX, refcnt;
+ unsigned long flags;
+
+ /* allocate PC with the minimun refcount */
+ for (i = 0; i < cqdma->dma_channels; ++i) {
+ refcnt = refcount_read(&cqdma->pc[i]->refcnt);
+ if (refcnt < min_refcnt) {
+ pc = cqdma->pc[i];
+ min_refcnt = refcnt;
+ }
+ }
+
+ if (!pc)
+ return -ENOSPC;
+
+ spin_lock_irqsave(&pc->lock, flags);
+
+ if (!refcount_read(&pc->refcnt)) {
+ /* allocate PC when the refcount is zero */
+ mtk_cqdma_hard_reset(pc);
+
+ /* enable interrupt for this PC */
+ mtk_dma_set(pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);
+
+ /*
+ * refcount_inc would complain increment on 0; use-after-free.
+ * Thus, we need to explicitly set it as 1 initially.
+ */
+ refcount_set(&pc->refcnt, 1);
+ } else {
+ refcount_inc(&pc->refcnt);
+ }
+
+ spin_unlock_irqrestore(&pc->lock, flags);
+
+ vc->pc = pc;
+
+ return 0;
+}
+
+static void mtk_cqdma_free_chan_resources(struct dma_chan *c)
+{
+ struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
+ unsigned long flags;
+
+ /* free all descriptors in all lists on the VC */
+ mtk_cqdma_terminate_all(c);
+
+ spin_lock_irqsave(&cvc->pc->lock, flags);
+
+ /* PC is not freed until there is no VC mapped to it */
+ if (refcount_dec_and_test(&cvc->pc->refcnt)) {
+ /* start the flush operation and stop the engine */
+ mtk_dma_set(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);
+
+ /* wait for the completion of flush operation */
+ if (mtk_cqdma_poll_engine_done(cvc->pc, true) < 0)
+ dev_err(cqdma2dev(to_cqdma_dev(c)), "cqdma flush timeout\n");
+
+ /* clear the flush bit and interrupt flag */
+ mtk_dma_clr(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);
+ mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_FLAG,
+ MTK_CQDMA_INT_FLAG_BIT);
+
+ /* disable interrupt for this PC */
+ mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);
+ }
+
+ spin_unlock_irqrestore(&cvc->pc->lock, flags);
+}
+
+static int mtk_cqdma_hw_init(struct mtk_cqdma_device *cqdma)
+{
+ unsigned long flags;
+ int err;
+ u32 i;
+
+ pm_runtime_enable(cqdma2dev(cqdma));
+ pm_runtime_get_sync(cqdma2dev(cqdma));
+
+ err = clk_prepare_enable(cqdma->clk);
+
+ if (err) {
+ pm_runtime_put_sync(cqdma2dev(cqdma));
+ pm_runtime_disable(cqdma2dev(cqdma));
+ return err;
+ }
+
+ /* reset all PCs */
+ for (i = 0; i < cqdma->dma_channels; ++i) {
+ spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
+ if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0) {
+ dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");
+ spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
+
+ clk_disable_unprepare(cqdma->clk);
+ pm_runtime_put_sync(cqdma2dev(cqdma));
+ pm_runtime_disable(cqdma2dev(cqdma));
+ return -EINVAL;
+ }
+ spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
+ }
+
+ return 0;
+}
+
+static void mtk_cqdma_hw_deinit(struct mtk_cqdma_device *cqdma)
+{
+ unsigned long flags;
+ u32 i;
+
+ /* reset all PCs */
+ for (i = 0; i < cqdma->dma_channels; ++i) {
+ spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
+ if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0)
+ dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");
+ spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
+ }
+
+ clk_disable_unprepare(cqdma->clk);
+
+ pm_runtime_put_sync(cqdma2dev(cqdma));
+ pm_runtime_disable(cqdma2dev(cqdma));
+}
+
+static const struct of_device_id mtk_cqdma_match[] = {
+ { .compatible = "mediatek,mt6765-cqdma" },
+ { /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, mtk_cqdma_match);
+
+static int mtk_cqdma_probe(struct platform_device *pdev)
+{
+ struct mtk_cqdma_device *cqdma;
+ struct mtk_cqdma_vchan *vc;
+ struct dma_device *dd;
+ struct resource *res;
+ int err;
+ u32 i;
+
+ cqdma = devm_kzalloc(&pdev->dev, sizeof(*cqdma), GFP_KERNEL);
+ if (!cqdma)
+ return -ENOMEM;
+
+ dd = &cqdma->ddev;
+
+ cqdma->clk = devm_clk_get(&pdev->dev, "cqdma");
+ if (IS_ERR(cqdma->clk)) {
+ dev_err(&pdev->dev, "No clock for %s\n",
+ dev_name(&pdev->dev));
+ return PTR_ERR(cqdma->clk);
+ }
+
+ dma_cap_set(DMA_MEMCPY, dd->cap_mask);
+
+ dd->copy_align = MTK_CQDMA_ALIGN_SIZE;
+ dd->device_alloc_chan_resources = mtk_cqdma_alloc_chan_resources;
+ dd->device_free_chan_resources = mtk_cqdma_free_chan_resources;
+ dd->device_tx_status = mtk_cqdma_tx_status;
+ dd->device_issue_pending = mtk_cqdma_issue_pending;
+ dd->device_prep_dma_memcpy = mtk_cqdma_prep_dma_memcpy;
+ dd->device_terminate_all = mtk_cqdma_terminate_all;
+ dd->src_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;
+ dd->dst_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;
+ dd->directions = BIT(DMA_MEM_TO_MEM);
+ dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
+ dd->dev = &pdev->dev;
+ INIT_LIST_HEAD(&dd->channels);
+
+ if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
+ "dma-requests",
+ &cqdma->dma_requests)) {
+ dev_info(&pdev->dev,
+ "Using %u as missing dma-requests property\n",
+ MTK_CQDMA_NR_VCHANS);
+
+ cqdma->dma_requests = MTK_CQDMA_NR_VCHANS;
+ }
+
+ if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
+ "dma-channels",
+ &cqdma->dma_channels)) {
+ dev_info(&pdev->dev,
+ "Using %u as missing dma-channels property\n",
+ MTK_CQDMA_NR_PCHANS);
+
+ cqdma->dma_channels = MTK_CQDMA_NR_PCHANS;
+ }
+
+ cqdma->pc = devm_kcalloc(&pdev->dev, cqdma->dma_channels,
+ sizeof(*cqdma->pc), GFP_KERNEL);
+ if (!cqdma->pc)
+ return -ENOMEM;
+
+ /* initialization for PCs */
+ for (i = 0; i < cqdma->dma_channels; ++i) {
+ cqdma->pc[i] = devm_kcalloc(&pdev->dev, 1,
+ sizeof(**cqdma->pc), GFP_KERNEL);
+ if (!cqdma->pc[i])
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&cqdma->pc[i]->queue);
+ spin_lock_init(&cqdma->pc[i]->lock);
+ refcount_set(&cqdma->pc[i]->refcnt, 0);
+ cqdma->pc[i]->base = devm_platform_ioremap_resource(pdev, i);
+ if (IS_ERR(cqdma->pc[i]->base))
+ return PTR_ERR(cqdma->pc[i]->base);
+
+ /* allocate IRQ resource */
+ res = platform_get_resource(pdev, IORESOURCE_IRQ, i);
+ if (!res) {
+ dev_err(&pdev->dev, "No irq resource for %s\n",
+ dev_name(&pdev->dev));
+ return -EINVAL;
+ }
+ cqdma->pc[i]->irq = res->start;
+
+ err = devm_request_irq(&pdev->dev, cqdma->pc[i]->irq,
+ mtk_cqdma_irq, 0, dev_name(&pdev->dev),
+ cqdma);
+ if (err) {
+ dev_err(&pdev->dev,
+ "request_irq failed with err %d\n", err);
+ return -EINVAL;
+ }
+ }
+
+ /* allocate resource for VCs */
+ cqdma->vc = devm_kcalloc(&pdev->dev, cqdma->dma_requests,
+ sizeof(*cqdma->vc), GFP_KERNEL);
+ if (!cqdma->vc)
+ return -ENOMEM;
+
+ for (i = 0; i < cqdma->dma_requests; i++) {
+ vc = &cqdma->vc[i];
+ vc->vc.desc_free = mtk_cqdma_vdesc_free;
+ vchan_init(&vc->vc, dd);
+ init_completion(&vc->issue_completion);
+ }
+
+ err = dma_async_device_register(dd);
+ if (err)
+ return err;
+
+ err = of_dma_controller_register(pdev->dev.of_node,
+ of_dma_xlate_by_chan_id, cqdma);
+ if (err) {
+ dev_err(&pdev->dev,
+ "MediaTek CQDMA OF registration failed %d\n", err);
+ goto err_unregister;
+ }
+
+ err = mtk_cqdma_hw_init(cqdma);
+ if (err) {
+ dev_err(&pdev->dev,
+ "MediaTek CQDMA HW initialization failed %d\n", err);
+ goto err_unregister;
+ }
+
+ platform_set_drvdata(pdev, cqdma);
+
+ /* initialize tasklet for each PC */
+ for (i = 0; i < cqdma->dma_channels; ++i)
+ tasklet_setup(&cqdma->pc[i]->tasklet, mtk_cqdma_tasklet_cb);
+
+ dev_info(&pdev->dev, "MediaTek CQDMA driver registered\n");
+
+ return 0;
+
+err_unregister:
+ dma_async_device_unregister(dd);
+
+ return err;
+}
+
+static int mtk_cqdma_remove(struct platform_device *pdev)
+{
+ struct mtk_cqdma_device *cqdma = platform_get_drvdata(pdev);
+ struct mtk_cqdma_vchan *vc;
+ unsigned long flags;
+ int i;
+
+ /* kill VC task */
+ for (i = 0; i < cqdma->dma_requests; i++) {
+ vc = &cqdma->vc[i];
+
+ list_del(&vc->vc.chan.device_node);
+ tasklet_kill(&vc->vc.task);
+ }
+
+ /* disable interrupt */
+ for (i = 0; i < cqdma->dma_channels; i++) {
+ spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
+ mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_EN,
+ MTK_CQDMA_INT_EN_BIT);
+ spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
+
+ /* Waits for any pending IRQ handlers to complete */
+ synchronize_irq(cqdma->pc[i]->irq);
+
+ tasklet_kill(&cqdma->pc[i]->tasklet);
+ }
+
+ /* disable hardware */
+ mtk_cqdma_hw_deinit(cqdma);
+
+ dma_async_device_unregister(&cqdma->ddev);
+ of_dma_controller_free(pdev->dev.of_node);
+
+ return 0;
+}
+
+static struct platform_driver mtk_cqdma_driver = {
+ .probe = mtk_cqdma_probe,
+ .remove = mtk_cqdma_remove,
+ .driver = {
+ .name = KBUILD_MODNAME,
+ .of_match_table = mtk_cqdma_match,
+ },
+};
+module_platform_driver(mtk_cqdma_driver);
+
+MODULE_DESCRIPTION("MediaTek CQDMA Controller Driver");
+MODULE_AUTHOR("Shun-Chih Yu <shun-chih.yu@mediatek.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/dma/mediatek/mtk-hsdma.c b/drivers/dma/mediatek/mtk-hsdma.c
new file mode 100644
index 000000000..6ad8afbb9
--- /dev/null
+++ b/drivers/dma/mediatek/mtk-hsdma.c
@@ -0,0 +1,1059 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2017-2018 MediaTek Inc.
+
+/*
+ * Driver for MediaTek High-Speed DMA Controller
+ *
+ * Author: Sean Wang <sean.wang@mediatek.com>
+ *
+ */
+
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/iopoll.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/of_dma.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/refcount.h>
+#include <linux/slab.h>
+
+#include "../virt-dma.h"
+
+#define MTK_HSDMA_USEC_POLL 20
+#define MTK_HSDMA_TIMEOUT_POLL 200000
+#define MTK_HSDMA_DMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
+
+/* The default number of virtual channel */
+#define MTK_HSDMA_NR_VCHANS 3
+
+/* Only one physical channel supported */
+#define MTK_HSDMA_NR_MAX_PCHANS 1
+
+/* Macro for physical descriptor (PD) manipulation */
+/* The number of PD which must be 2 of power */
+#define MTK_DMA_SIZE 64
+#define MTK_HSDMA_NEXT_DESP_IDX(x, y) (((x) + 1) & ((y) - 1))
+#define MTK_HSDMA_LAST_DESP_IDX(x, y) (((x) - 1) & ((y) - 1))
+#define MTK_HSDMA_MAX_LEN 0x3f80
+#define MTK_HSDMA_ALIGN_SIZE 4
+#define MTK_HSDMA_PLEN_MASK 0x3fff
+#define MTK_HSDMA_DESC_PLEN(x) (((x) & MTK_HSDMA_PLEN_MASK) << 16)
+#define MTK_HSDMA_DESC_PLEN_GET(x) (((x) >> 16) & MTK_HSDMA_PLEN_MASK)
+
+/* Registers for underlying ring manipulation */
+#define MTK_HSDMA_TX_BASE 0x0
+#define MTK_HSDMA_TX_CNT 0x4
+#define MTK_HSDMA_TX_CPU 0x8
+#define MTK_HSDMA_TX_DMA 0xc
+#define MTK_HSDMA_RX_BASE 0x100
+#define MTK_HSDMA_RX_CNT 0x104
+#define MTK_HSDMA_RX_CPU 0x108
+#define MTK_HSDMA_RX_DMA 0x10c
+
+/* Registers for global setup */
+#define MTK_HSDMA_GLO 0x204
+#define MTK_HSDMA_GLO_MULTI_DMA BIT(10)
+#define MTK_HSDMA_TX_WB_DDONE BIT(6)
+#define MTK_HSDMA_BURST_64BYTES (0x2 << 4)
+#define MTK_HSDMA_GLO_RX_BUSY BIT(3)
+#define MTK_HSDMA_GLO_RX_DMA BIT(2)
+#define MTK_HSDMA_GLO_TX_BUSY BIT(1)
+#define MTK_HSDMA_GLO_TX_DMA BIT(0)
+#define MTK_HSDMA_GLO_DMA (MTK_HSDMA_GLO_TX_DMA | \
+ MTK_HSDMA_GLO_RX_DMA)
+#define MTK_HSDMA_GLO_BUSY (MTK_HSDMA_GLO_RX_BUSY | \
+ MTK_HSDMA_GLO_TX_BUSY)
+#define MTK_HSDMA_GLO_DEFAULT (MTK_HSDMA_GLO_TX_DMA | \
+ MTK_HSDMA_GLO_RX_DMA | \
+ MTK_HSDMA_TX_WB_DDONE | \
+ MTK_HSDMA_BURST_64BYTES | \
+ MTK_HSDMA_GLO_MULTI_DMA)
+
+/* Registers for reset */
+#define MTK_HSDMA_RESET 0x208
+#define MTK_HSDMA_RST_TX BIT(0)
+#define MTK_HSDMA_RST_RX BIT(16)
+
+/* Registers for interrupt control */
+#define MTK_HSDMA_DLYINT 0x20c
+#define MTK_HSDMA_RXDLY_INT_EN BIT(15)
+
+/* Interrupt fires when the pending number's more than the specified */
+#define MTK_HSDMA_RXMAX_PINT(x) (((x) & 0x7f) << 8)
+
+/* Interrupt fires when the pending time's more than the specified in 20 us */
+#define MTK_HSDMA_RXMAX_PTIME(x) ((x) & 0x7f)
+#define MTK_HSDMA_DLYINT_DEFAULT (MTK_HSDMA_RXDLY_INT_EN | \
+ MTK_HSDMA_RXMAX_PINT(20) | \
+ MTK_HSDMA_RXMAX_PTIME(20))
+#define MTK_HSDMA_INT_STATUS 0x220
+#define MTK_HSDMA_INT_ENABLE 0x228
+#define MTK_HSDMA_INT_RXDONE BIT(16)
+
+enum mtk_hsdma_vdesc_flag {
+ MTK_HSDMA_VDESC_FINISHED = 0x01,
+};
+
+#define IS_MTK_HSDMA_VDESC_FINISHED(x) ((x) == MTK_HSDMA_VDESC_FINISHED)
+
+/**
+ * struct mtk_hsdma_pdesc - This is the struct holding info describing physical
+ * descriptor (PD) and its placement must be kept at
+ * 4-bytes alignment in little endian order.
+ * @desc1: | The control pad used to indicate hardware how to
+ * @desc2: | deal with the descriptor such as source and
+ * @desc3: | destination address and data length. The maximum
+ * @desc4: | data length each pdesc can handle is 0x3f80 bytes
+ */
+struct mtk_hsdma_pdesc {
+ __le32 desc1;
+ __le32 desc2;
+ __le32 desc3;
+ __le32 desc4;
+} __packed __aligned(4);
+
+/**
+ * struct mtk_hsdma_vdesc - This is the struct holding info describing virtual
+ * descriptor (VD)
+ * @vd: An instance for struct virt_dma_desc
+ * @len: The total data size device wants to move
+ * @residue: The remaining data size device will move
+ * @dest: The destination address device wants to move to
+ * @src: The source address device wants to move from
+ */
+struct mtk_hsdma_vdesc {
+ struct virt_dma_desc vd;
+ size_t len;
+ size_t residue;
+ dma_addr_t dest;
+ dma_addr_t src;
+};
+
+/**
+ * struct mtk_hsdma_cb - This is the struct holding extra info required for RX
+ * ring to know what relevant VD the the PD is being
+ * mapped to.
+ * @vd: Pointer to the relevant VD.
+ * @flag: Flag indicating what action should be taken when VD
+ * is completed.
+ */
+struct mtk_hsdma_cb {
+ struct virt_dma_desc *vd;
+ enum mtk_hsdma_vdesc_flag flag;
+};
+
+/**
+ * struct mtk_hsdma_ring - This struct holds info describing underlying ring
+ * space
+ * @txd: The descriptor TX ring which describes DMA source
+ * information
+ * @rxd: The descriptor RX ring which describes DMA
+ * destination information
+ * @cb: The extra information pointed at by RX ring
+ * @tphys: The physical addr of TX ring
+ * @rphys: The physical addr of RX ring
+ * @cur_tptr: Pointer to the next free descriptor used by the host
+ * @cur_rptr: Pointer to the last done descriptor by the device
+ */
+struct mtk_hsdma_ring {
+ struct mtk_hsdma_pdesc *txd;
+ struct mtk_hsdma_pdesc *rxd;
+ struct mtk_hsdma_cb *cb;
+ dma_addr_t tphys;
+ dma_addr_t rphys;
+ u16 cur_tptr;
+ u16 cur_rptr;
+};
+
+/**
+ * struct mtk_hsdma_pchan - This is the struct holding info describing physical
+ * channel (PC)
+ * @ring: An instance for the underlying ring
+ * @sz_ring: Total size allocated for the ring
+ * @nr_free: Total number of free rooms in the ring. It would
+ * be accessed and updated frequently between IRQ
+ * context and user context to reflect whether ring
+ * can accept requests from VD.
+ */
+struct mtk_hsdma_pchan {
+ struct mtk_hsdma_ring ring;
+ size_t sz_ring;
+ atomic_t nr_free;
+};
+
+/**
+ * struct mtk_hsdma_vchan - This is the struct holding info describing virtual
+ * channel (VC)
+ * @vc: An instance for struct virt_dma_chan
+ * @issue_completion: The wait for all issued descriptors completited
+ * @issue_synchronize: Bool indicating channel synchronization starts
+ * @desc_hw_processing: List those descriptors the hardware is processing,
+ * which is protected by vc.lock
+ */
+struct mtk_hsdma_vchan {
+ struct virt_dma_chan vc;
+ struct completion issue_completion;
+ bool issue_synchronize;
+ struct list_head desc_hw_processing;
+};
+
+/**
+ * struct mtk_hsdma_soc - This is the struct holding differences among SoCs
+ * @ddone: Bit mask for DDONE
+ * @ls0: Bit mask for LS0
+ */
+struct mtk_hsdma_soc {
+ __le32 ddone;
+ __le32 ls0;
+};
+
+/**
+ * struct mtk_hsdma_device - This is the struct holding info describing HSDMA
+ * device
+ * @ddev: An instance for struct dma_device
+ * @base: The mapped register I/O base
+ * @clk: The clock that device internal is using
+ * @irq: The IRQ that device are using
+ * @dma_requests: The number of VCs the device supports to
+ * @vc: The pointer to all available VCs
+ * @pc: The pointer to the underlying PC
+ * @pc_refcnt: Track how many VCs are using the PC
+ * @lock: Lock protect agaisting multiple VCs access PC
+ * @soc: The pointer to area holding differences among
+ * vaious platform
+ */
+struct mtk_hsdma_device {
+ struct dma_device ddev;
+ void __iomem *base;
+ struct clk *clk;
+ u32 irq;
+
+ u32 dma_requests;
+ struct mtk_hsdma_vchan *vc;
+ struct mtk_hsdma_pchan *pc;
+ refcount_t pc_refcnt;
+
+ /* Lock used to protect against multiple VCs access PC */
+ spinlock_t lock;
+
+ const struct mtk_hsdma_soc *soc;
+};
+
+static struct mtk_hsdma_device *to_hsdma_dev(struct dma_chan *chan)
+{
+ return container_of(chan->device, struct mtk_hsdma_device, ddev);
+}
+
+static inline struct mtk_hsdma_vchan *to_hsdma_vchan(struct dma_chan *chan)
+{
+ return container_of(chan, struct mtk_hsdma_vchan, vc.chan);
+}
+
+static struct mtk_hsdma_vdesc *to_hsdma_vdesc(struct virt_dma_desc *vd)
+{
+ return container_of(vd, struct mtk_hsdma_vdesc, vd);
+}
+
+static struct device *hsdma2dev(struct mtk_hsdma_device *hsdma)
+{
+ return hsdma->ddev.dev;
+}
+
+static u32 mtk_dma_read(struct mtk_hsdma_device *hsdma, u32 reg)
+{
+ return readl(hsdma->base + reg);
+}
+
+static void mtk_dma_write(struct mtk_hsdma_device *hsdma, u32 reg, u32 val)
+{
+ writel(val, hsdma->base + reg);
+}
+
+static void mtk_dma_rmw(struct mtk_hsdma_device *hsdma, u32 reg,
+ u32 mask, u32 set)
+{
+ u32 val;
+
+ val = mtk_dma_read(hsdma, reg);
+ val &= ~mask;
+ val |= set;
+ mtk_dma_write(hsdma, reg, val);
+}
+
+static void mtk_dma_set(struct mtk_hsdma_device *hsdma, u32 reg, u32 val)
+{
+ mtk_dma_rmw(hsdma, reg, 0, val);
+}
+
+static void mtk_dma_clr(struct mtk_hsdma_device *hsdma, u32 reg, u32 val)
+{
+ mtk_dma_rmw(hsdma, reg, val, 0);
+}
+
+static void mtk_hsdma_vdesc_free(struct virt_dma_desc *vd)
+{
+ kfree(container_of(vd, struct mtk_hsdma_vdesc, vd));
+}
+
+static int mtk_hsdma_busy_wait(struct mtk_hsdma_device *hsdma)
+{
+ u32 status = 0;
+
+ return readl_poll_timeout(hsdma->base + MTK_HSDMA_GLO, status,
+ !(status & MTK_HSDMA_GLO_BUSY),
+ MTK_HSDMA_USEC_POLL,
+ MTK_HSDMA_TIMEOUT_POLL);
+}
+
+static int mtk_hsdma_alloc_pchan(struct mtk_hsdma_device *hsdma,
+ struct mtk_hsdma_pchan *pc)
+{
+ struct mtk_hsdma_ring *ring = &pc->ring;
+ int err;
+
+ memset(pc, 0, sizeof(*pc));
+
+ /*
+ * Allocate ring space where [0 ... MTK_DMA_SIZE - 1] is for TX ring
+ * and [MTK_DMA_SIZE ... 2 * MTK_DMA_SIZE - 1] is for RX ring.
+ */
+ pc->sz_ring = 2 * MTK_DMA_SIZE * sizeof(*ring->txd);
+ ring->txd = dma_alloc_coherent(hsdma2dev(hsdma), pc->sz_ring,
+ &ring->tphys, GFP_NOWAIT);
+ if (!ring->txd)
+ return -ENOMEM;
+
+ ring->rxd = &ring->txd[MTK_DMA_SIZE];
+ ring->rphys = ring->tphys + MTK_DMA_SIZE * sizeof(*ring->txd);
+ ring->cur_tptr = 0;
+ ring->cur_rptr = MTK_DMA_SIZE - 1;
+
+ ring->cb = kcalloc(MTK_DMA_SIZE, sizeof(*ring->cb), GFP_NOWAIT);
+ if (!ring->cb) {
+ err = -ENOMEM;
+ goto err_free_dma;
+ }
+
+ atomic_set(&pc->nr_free, MTK_DMA_SIZE - 1);
+
+ /* Disable HSDMA and wait for the completion */
+ mtk_dma_clr(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA);
+ err = mtk_hsdma_busy_wait(hsdma);
+ if (err)
+ goto err_free_cb;
+
+ /* Reset */
+ mtk_dma_set(hsdma, MTK_HSDMA_RESET,
+ MTK_HSDMA_RST_TX | MTK_HSDMA_RST_RX);
+ mtk_dma_clr(hsdma, MTK_HSDMA_RESET,
+ MTK_HSDMA_RST_TX | MTK_HSDMA_RST_RX);
+
+ /* Setup HSDMA initial pointer in the ring */
+ mtk_dma_write(hsdma, MTK_HSDMA_TX_BASE, ring->tphys);
+ mtk_dma_write(hsdma, MTK_HSDMA_TX_CNT, MTK_DMA_SIZE);
+ mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, ring->cur_tptr);
+ mtk_dma_write(hsdma, MTK_HSDMA_TX_DMA, 0);
+ mtk_dma_write(hsdma, MTK_HSDMA_RX_BASE, ring->rphys);
+ mtk_dma_write(hsdma, MTK_HSDMA_RX_CNT, MTK_DMA_SIZE);
+ mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, ring->cur_rptr);
+ mtk_dma_write(hsdma, MTK_HSDMA_RX_DMA, 0);
+
+ /* Enable HSDMA */
+ mtk_dma_set(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA);
+
+ /* Setup delayed interrupt */
+ mtk_dma_write(hsdma, MTK_HSDMA_DLYINT, MTK_HSDMA_DLYINT_DEFAULT);
+
+ /* Enable interrupt */
+ mtk_dma_set(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);
+
+ return 0;
+
+err_free_cb:
+ kfree(ring->cb);
+
+err_free_dma:
+ dma_free_coherent(hsdma2dev(hsdma),
+ pc->sz_ring, ring->txd, ring->tphys);
+ return err;
+}
+
+static void mtk_hsdma_free_pchan(struct mtk_hsdma_device *hsdma,
+ struct mtk_hsdma_pchan *pc)
+{
+ struct mtk_hsdma_ring *ring = &pc->ring;
+
+ /* Disable HSDMA and then wait for the completion */
+ mtk_dma_clr(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA);
+ mtk_hsdma_busy_wait(hsdma);
+
+ /* Reset pointer in the ring */
+ mtk_dma_clr(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);
+ mtk_dma_write(hsdma, MTK_HSDMA_TX_BASE, 0);
+ mtk_dma_write(hsdma, MTK_HSDMA_TX_CNT, 0);
+ mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, 0);
+ mtk_dma_write(hsdma, MTK_HSDMA_RX_BASE, 0);
+ mtk_dma_write(hsdma, MTK_HSDMA_RX_CNT, 0);
+ mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, MTK_DMA_SIZE - 1);
+
+ kfree(ring->cb);
+
+ dma_free_coherent(hsdma2dev(hsdma),
+ pc->sz_ring, ring->txd, ring->tphys);
+}
+
+static int mtk_hsdma_issue_pending_vdesc(struct mtk_hsdma_device *hsdma,
+ struct mtk_hsdma_pchan *pc,
+ struct mtk_hsdma_vdesc *hvd)
+{
+ struct mtk_hsdma_ring *ring = &pc->ring;
+ struct mtk_hsdma_pdesc *txd, *rxd;
+ u16 reserved, prev, tlen, num_sgs;
+ unsigned long flags;
+
+ /* Protect against PC is accessed by multiple VCs simultaneously */
+ spin_lock_irqsave(&hsdma->lock, flags);
+
+ /*
+ * Reserve rooms, where pc->nr_free is used to track how many free
+ * rooms in the ring being updated in user and IRQ context.
+ */
+ num_sgs = DIV_ROUND_UP(hvd->len, MTK_HSDMA_MAX_LEN);
+ reserved = min_t(u16, num_sgs, atomic_read(&pc->nr_free));
+
+ if (!reserved) {
+ spin_unlock_irqrestore(&hsdma->lock, flags);
+ return -ENOSPC;
+ }
+
+ atomic_sub(reserved, &pc->nr_free);
+
+ while (reserved--) {
+ /* Limit size by PD capability for valid data moving */
+ tlen = (hvd->len > MTK_HSDMA_MAX_LEN) ?
+ MTK_HSDMA_MAX_LEN : hvd->len;
+
+ /*
+ * Setup PDs using the remaining VD info mapped on those
+ * reserved rooms. And since RXD is shared memory between the
+ * host and the device allocated by dma_alloc_coherent call,
+ * the helper macro WRITE_ONCE can ensure the data written to
+ * RAM would really happens.
+ */
+ txd = &ring->txd[ring->cur_tptr];
+ WRITE_ONCE(txd->desc1, hvd->src);
+ WRITE_ONCE(txd->desc2,
+ hsdma->soc->ls0 | MTK_HSDMA_DESC_PLEN(tlen));
+
+ rxd = &ring->rxd[ring->cur_tptr];
+ WRITE_ONCE(rxd->desc1, hvd->dest);
+ WRITE_ONCE(rxd->desc2, MTK_HSDMA_DESC_PLEN(tlen));
+
+ /* Associate VD, the PD belonged to */
+ ring->cb[ring->cur_tptr].vd = &hvd->vd;
+
+ /* Move forward the pointer of TX ring */
+ ring->cur_tptr = MTK_HSDMA_NEXT_DESP_IDX(ring->cur_tptr,
+ MTK_DMA_SIZE);
+
+ /* Update VD with remaining data */
+ hvd->src += tlen;
+ hvd->dest += tlen;
+ hvd->len -= tlen;
+ }
+
+ /*
+ * Tagging flag for the last PD for VD will be responsible for
+ * completing VD.
+ */
+ if (!hvd->len) {
+ prev = MTK_HSDMA_LAST_DESP_IDX(ring->cur_tptr, MTK_DMA_SIZE);
+ ring->cb[prev].flag = MTK_HSDMA_VDESC_FINISHED;
+ }
+
+ /* Ensure all changes indeed done before we're going on */
+ wmb();
+
+ /*
+ * Updating into hardware the pointer of TX ring lets HSDMA to take
+ * action for those pending PDs.
+ */
+ mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, ring->cur_tptr);
+
+ spin_unlock_irqrestore(&hsdma->lock, flags);
+
+ return 0;
+}
+
+static void mtk_hsdma_issue_vchan_pending(struct mtk_hsdma_device *hsdma,
+ struct mtk_hsdma_vchan *hvc)
+{
+ struct virt_dma_desc *vd, *vd2;
+ int err;
+
+ lockdep_assert_held(&hvc->vc.lock);
+
+ list_for_each_entry_safe(vd, vd2, &hvc->vc.desc_issued, node) {
+ struct mtk_hsdma_vdesc *hvd;
+
+ hvd = to_hsdma_vdesc(vd);
+
+ /* Map VD into PC and all VCs shares a single PC */
+ err = mtk_hsdma_issue_pending_vdesc(hsdma, hsdma->pc, hvd);
+
+ /*
+ * Move VD from desc_issued to desc_hw_processing when entire
+ * VD is fit into available PDs. Otherwise, the uncompleted
+ * VDs would stay in list desc_issued and then restart the
+ * processing as soon as possible once underlying ring space
+ * got freed.
+ */
+ if (err == -ENOSPC || hvd->len > 0)
+ break;
+
+ /*
+ * The extra list desc_hw_processing is used because
+ * hardware can't provide sufficient information allowing us
+ * to know what VDs are still working on the underlying ring.
+ * Through the additional list, it can help us to implement
+ * terminate_all, residue calculation and such thing needed
+ * to know detail descriptor status on the hardware.
+ */
+ list_move_tail(&vd->node, &hvc->desc_hw_processing);
+ }
+}
+
+static void mtk_hsdma_free_rooms_in_ring(struct mtk_hsdma_device *hsdma)
+{
+ struct mtk_hsdma_vchan *hvc;
+ struct mtk_hsdma_pdesc *rxd;
+ struct mtk_hsdma_vdesc *hvd;
+ struct mtk_hsdma_pchan *pc;
+ struct mtk_hsdma_cb *cb;
+ int i = MTK_DMA_SIZE;
+ __le32 desc2;
+ u32 status;
+ u16 next;
+
+ /* Read IRQ status */
+ status = mtk_dma_read(hsdma, MTK_HSDMA_INT_STATUS);
+ if (unlikely(!(status & MTK_HSDMA_INT_RXDONE)))
+ goto rx_done;
+
+ pc = hsdma->pc;
+
+ /*
+ * Using a fail-safe loop with iterations of up to MTK_DMA_SIZE to
+ * reclaim these finished descriptors: The most number of PDs the ISR
+ * can handle at one time shouldn't be more than MTK_DMA_SIZE so we
+ * take it as limited count instead of just using a dangerous infinite
+ * poll.
+ */
+ while (i--) {
+ next = MTK_HSDMA_NEXT_DESP_IDX(pc->ring.cur_rptr,
+ MTK_DMA_SIZE);
+ rxd = &pc->ring.rxd[next];
+
+ /*
+ * If MTK_HSDMA_DESC_DDONE is no specified, that means data
+ * moving for the PD is still under going.
+ */
+ desc2 = READ_ONCE(rxd->desc2);
+ if (!(desc2 & hsdma->soc->ddone))
+ break;
+
+ cb = &pc->ring.cb[next];
+ if (unlikely(!cb->vd)) {
+ dev_err(hsdma2dev(hsdma), "cb->vd cannot be null\n");
+ break;
+ }
+
+ /* Update residue of VD the associated PD belonged to */
+ hvd = to_hsdma_vdesc(cb->vd);
+ hvd->residue -= MTK_HSDMA_DESC_PLEN_GET(rxd->desc2);
+
+ /* Complete VD until the relevant last PD is finished */
+ if (IS_MTK_HSDMA_VDESC_FINISHED(cb->flag)) {
+ hvc = to_hsdma_vchan(cb->vd->tx.chan);
+
+ spin_lock(&hvc->vc.lock);
+
+ /* Remove VD from list desc_hw_processing */
+ list_del(&cb->vd->node);
+
+ /* Add VD into list desc_completed */
+ vchan_cookie_complete(cb->vd);
+
+ if (hvc->issue_synchronize &&
+ list_empty(&hvc->desc_hw_processing)) {
+ complete(&hvc->issue_completion);
+ hvc->issue_synchronize = false;
+ }
+ spin_unlock(&hvc->vc.lock);
+
+ cb->flag = 0;
+ }
+
+ cb->vd = 0;
+
+ /*
+ * Recycle the RXD with the helper WRITE_ONCE that can ensure
+ * data written into RAM would really happens.
+ */
+ WRITE_ONCE(rxd->desc1, 0);
+ WRITE_ONCE(rxd->desc2, 0);
+ pc->ring.cur_rptr = next;
+
+ /* Release rooms */
+ atomic_inc(&pc->nr_free);
+ }
+
+ /* Ensure all changes indeed done before we're going on */
+ wmb();
+
+ /* Update CPU pointer for those completed PDs */
+ mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, pc->ring.cur_rptr);
+
+ /*
+ * Acking the pending IRQ allows hardware no longer to keep the used
+ * IRQ line in certain trigger state when software has completed all
+ * the finished physical descriptors.
+ */
+ if (atomic_read(&pc->nr_free) >= MTK_DMA_SIZE - 1)
+ mtk_dma_write(hsdma, MTK_HSDMA_INT_STATUS, status);
+
+ /* ASAP handles pending VDs in all VCs after freeing some rooms */
+ for (i = 0; i < hsdma->dma_requests; i++) {
+ hvc = &hsdma->vc[i];
+ spin_lock(&hvc->vc.lock);
+ mtk_hsdma_issue_vchan_pending(hsdma, hvc);
+ spin_unlock(&hvc->vc.lock);
+ }
+
+rx_done:
+ /* All completed PDs are cleaned up, so enable interrupt again */
+ mtk_dma_set(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);
+}
+
+static irqreturn_t mtk_hsdma_irq(int irq, void *devid)
+{
+ struct mtk_hsdma_device *hsdma = devid;
+
+ /*
+ * Disable interrupt until all completed PDs are cleaned up in
+ * mtk_hsdma_free_rooms call.
+ */
+ mtk_dma_clr(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);
+
+ mtk_hsdma_free_rooms_in_ring(hsdma);
+
+ return IRQ_HANDLED;
+}
+
+static struct virt_dma_desc *mtk_hsdma_find_active_desc(struct dma_chan *c,
+ dma_cookie_t cookie)
+{
+ struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c);
+ struct virt_dma_desc *vd;
+
+ list_for_each_entry(vd, &hvc->desc_hw_processing, node)
+ if (vd->tx.cookie == cookie)
+ return vd;
+
+ list_for_each_entry(vd, &hvc->vc.desc_issued, node)
+ if (vd->tx.cookie == cookie)
+ return vd;
+
+ return NULL;
+}
+
+static enum dma_status mtk_hsdma_tx_status(struct dma_chan *c,
+ dma_cookie_t cookie,
+ struct dma_tx_state *txstate)
+{
+ struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c);
+ struct mtk_hsdma_vdesc *hvd;
+ struct virt_dma_desc *vd;
+ enum dma_status ret;
+ unsigned long flags;
+ size_t bytes = 0;
+
+ ret = dma_cookie_status(c, cookie, txstate);
+ if (ret == DMA_COMPLETE || !txstate)
+ return ret;
+
+ spin_lock_irqsave(&hvc->vc.lock, flags);
+ vd = mtk_hsdma_find_active_desc(c, cookie);
+ spin_unlock_irqrestore(&hvc->vc.lock, flags);
+
+ if (vd) {
+ hvd = to_hsdma_vdesc(vd);
+ bytes = hvd->residue;
+ }
+
+ dma_set_residue(txstate, bytes);
+
+ return ret;
+}
+
+static void mtk_hsdma_issue_pending(struct dma_chan *c)
+{
+ struct mtk_hsdma_device *hsdma = to_hsdma_dev(c);
+ struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c);
+ unsigned long flags;
+
+ spin_lock_irqsave(&hvc->vc.lock, flags);
+
+ if (vchan_issue_pending(&hvc->vc))
+ mtk_hsdma_issue_vchan_pending(hsdma, hvc);
+
+ spin_unlock_irqrestore(&hvc->vc.lock, flags);
+}
+
+static struct dma_async_tx_descriptor *
+mtk_hsdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest,
+ dma_addr_t src, size_t len, unsigned long flags)
+{
+ struct mtk_hsdma_vdesc *hvd;
+
+ hvd = kzalloc(sizeof(*hvd), GFP_NOWAIT);
+ if (!hvd)
+ return NULL;
+
+ hvd->len = len;
+ hvd->residue = len;
+ hvd->src = src;
+ hvd->dest = dest;
+
+ return vchan_tx_prep(to_virt_chan(c), &hvd->vd, flags);
+}
+
+static int mtk_hsdma_free_inactive_desc(struct dma_chan *c)
+{
+ struct virt_dma_chan *vc = to_virt_chan(c);
+ unsigned long flags;
+ LIST_HEAD(head);
+
+ spin_lock_irqsave(&vc->lock, flags);
+ list_splice_tail_init(&vc->desc_allocated, &head);
+ list_splice_tail_init(&vc->desc_submitted, &head);
+ list_splice_tail_init(&vc->desc_issued, &head);
+ spin_unlock_irqrestore(&vc->lock, flags);
+
+ /* At the point, we don't expect users put descriptor into VC again */
+ vchan_dma_desc_free_list(vc, &head);
+
+ return 0;
+}
+
+static void mtk_hsdma_free_active_desc(struct dma_chan *c)
+{
+ struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c);
+ bool sync_needed = false;
+
+ /*
+ * Once issue_synchronize is being set, which means once the hardware
+ * consumes all descriptors for the channel in the ring, the
+ * synchronization must be be notified immediately it is completed.
+ */
+ spin_lock(&hvc->vc.lock);
+ if (!list_empty(&hvc->desc_hw_processing)) {
+ hvc->issue_synchronize = true;
+ sync_needed = true;
+ }
+ spin_unlock(&hvc->vc.lock);
+
+ if (sync_needed)
+ wait_for_completion(&hvc->issue_completion);
+ /*
+ * At the point, we expect that all remaining descriptors in the ring
+ * for the channel should be all processing done.
+ */
+ WARN_ONCE(!list_empty(&hvc->desc_hw_processing),
+ "Desc pending still in list desc_hw_processing\n");
+
+ /* Free all descriptors in list desc_completed */
+ vchan_synchronize(&hvc->vc);
+
+ WARN_ONCE(!list_empty(&hvc->vc.desc_completed),
+ "Desc pending still in list desc_completed\n");
+}
+
+static int mtk_hsdma_terminate_all(struct dma_chan *c)
+{
+ /*
+ * Free pending descriptors not processed yet by hardware that have
+ * previously been submitted to the channel.
+ */
+ mtk_hsdma_free_inactive_desc(c);
+
+ /*
+ * However, the DMA engine doesn't provide any way to stop these
+ * descriptors being processed currently by hardware. The only way is
+ * to just waiting until these descriptors are all processed completely
+ * through mtk_hsdma_free_active_desc call.
+ */
+ mtk_hsdma_free_active_desc(c);
+
+ return 0;
+}
+
+static int mtk_hsdma_alloc_chan_resources(struct dma_chan *c)
+{
+ struct mtk_hsdma_device *hsdma = to_hsdma_dev(c);
+ int err;
+
+ /*
+ * Since HSDMA has only one PC, the resource for PC is being allocated
+ * when the first VC is being created and the other VCs would run on
+ * the same PC.
+ */
+ if (!refcount_read(&hsdma->pc_refcnt)) {
+ err = mtk_hsdma_alloc_pchan(hsdma, hsdma->pc);
+ if (err)
+ return err;
+ /*
+ * refcount_inc would complain increment on 0; use-after-free.
+ * Thus, we need to explicitly set it as 1 initially.
+ */
+ refcount_set(&hsdma->pc_refcnt, 1);
+ } else {
+ refcount_inc(&hsdma->pc_refcnt);
+ }
+
+ return 0;
+}
+
+static void mtk_hsdma_free_chan_resources(struct dma_chan *c)
+{
+ struct mtk_hsdma_device *hsdma = to_hsdma_dev(c);
+
+ /* Free all descriptors in all lists on the VC */
+ mtk_hsdma_terminate_all(c);
+
+ /* The resource for PC is not freed until all the VCs are destroyed */
+ if (!refcount_dec_and_test(&hsdma->pc_refcnt))
+ return;
+
+ mtk_hsdma_free_pchan(hsdma, hsdma->pc);
+}
+
+static int mtk_hsdma_hw_init(struct mtk_hsdma_device *hsdma)
+{
+ int err;
+
+ pm_runtime_enable(hsdma2dev(hsdma));
+ pm_runtime_get_sync(hsdma2dev(hsdma));
+
+ err = clk_prepare_enable(hsdma->clk);
+ if (err)
+ return err;
+
+ mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, 0);
+ mtk_dma_write(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DEFAULT);
+
+ return 0;
+}
+
+static int mtk_hsdma_hw_deinit(struct mtk_hsdma_device *hsdma)
+{
+ mtk_dma_write(hsdma, MTK_HSDMA_GLO, 0);
+
+ clk_disable_unprepare(hsdma->clk);
+
+ pm_runtime_put_sync(hsdma2dev(hsdma));
+ pm_runtime_disable(hsdma2dev(hsdma));
+
+ return 0;
+}
+
+static const struct mtk_hsdma_soc mt7623_soc = {
+ .ddone = BIT(31),
+ .ls0 = BIT(30),
+};
+
+static const struct mtk_hsdma_soc mt7622_soc = {
+ .ddone = BIT(15),
+ .ls0 = BIT(14),
+};
+
+static const struct of_device_id mtk_hsdma_match[] = {
+ { .compatible = "mediatek,mt7623-hsdma", .data = &mt7623_soc},
+ { .compatible = "mediatek,mt7622-hsdma", .data = &mt7622_soc},
+ { /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, mtk_hsdma_match);
+
+static int mtk_hsdma_probe(struct platform_device *pdev)
+{
+ struct mtk_hsdma_device *hsdma;
+ struct mtk_hsdma_vchan *vc;
+ struct dma_device *dd;
+ struct resource *res;
+ int i, err;
+
+ hsdma = devm_kzalloc(&pdev->dev, sizeof(*hsdma), GFP_KERNEL);
+ if (!hsdma)
+ return -ENOMEM;
+
+ dd = &hsdma->ddev;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ hsdma->base = devm_ioremap_resource(&pdev->dev, res);
+ if (IS_ERR(hsdma->base))
+ return PTR_ERR(hsdma->base);
+
+ hsdma->soc = of_device_get_match_data(&pdev->dev);
+ if (!hsdma->soc) {
+ dev_err(&pdev->dev, "No device match found\n");
+ return -ENODEV;
+ }
+
+ hsdma->clk = devm_clk_get(&pdev->dev, "hsdma");
+ if (IS_ERR(hsdma->clk)) {
+ dev_err(&pdev->dev, "No clock for %s\n",
+ dev_name(&pdev->dev));
+ return PTR_ERR(hsdma->clk);
+ }
+
+ res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
+ if (!res) {
+ dev_err(&pdev->dev, "No irq resource for %s\n",
+ dev_name(&pdev->dev));
+ return -EINVAL;
+ }
+ hsdma->irq = res->start;
+
+ refcount_set(&hsdma->pc_refcnt, 0);
+ spin_lock_init(&hsdma->lock);
+
+ dma_cap_set(DMA_MEMCPY, dd->cap_mask);
+
+ dd->copy_align = MTK_HSDMA_ALIGN_SIZE;
+ dd->device_alloc_chan_resources = mtk_hsdma_alloc_chan_resources;
+ dd->device_free_chan_resources = mtk_hsdma_free_chan_resources;
+ dd->device_tx_status = mtk_hsdma_tx_status;
+ dd->device_issue_pending = mtk_hsdma_issue_pending;
+ dd->device_prep_dma_memcpy = mtk_hsdma_prep_dma_memcpy;
+ dd->device_terminate_all = mtk_hsdma_terminate_all;
+ dd->src_addr_widths = MTK_HSDMA_DMA_BUSWIDTHS;
+ dd->dst_addr_widths = MTK_HSDMA_DMA_BUSWIDTHS;
+ dd->directions = BIT(DMA_MEM_TO_MEM);
+ dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
+ dd->dev = &pdev->dev;
+ INIT_LIST_HEAD(&dd->channels);
+
+ hsdma->dma_requests = MTK_HSDMA_NR_VCHANS;
+ if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
+ "dma-requests",
+ &hsdma->dma_requests)) {
+ dev_info(&pdev->dev,
+ "Using %u as missing dma-requests property\n",
+ MTK_HSDMA_NR_VCHANS);
+ }
+
+ hsdma->pc = devm_kcalloc(&pdev->dev, MTK_HSDMA_NR_MAX_PCHANS,
+ sizeof(*hsdma->pc), GFP_KERNEL);
+ if (!hsdma->pc)
+ return -ENOMEM;
+
+ hsdma->vc = devm_kcalloc(&pdev->dev, hsdma->dma_requests,
+ sizeof(*hsdma->vc), GFP_KERNEL);
+ if (!hsdma->vc)
+ return -ENOMEM;
+
+ for (i = 0; i < hsdma->dma_requests; i++) {
+ vc = &hsdma->vc[i];
+ vc->vc.desc_free = mtk_hsdma_vdesc_free;
+ vchan_init(&vc->vc, dd);
+ init_completion(&vc->issue_completion);
+ INIT_LIST_HEAD(&vc->desc_hw_processing);
+ }
+
+ err = dma_async_device_register(dd);
+ if (err)
+ return err;
+
+ err = of_dma_controller_register(pdev->dev.of_node,
+ of_dma_xlate_by_chan_id, hsdma);
+ if (err) {
+ dev_err(&pdev->dev,
+ "MediaTek HSDMA OF registration failed %d\n", err);
+ goto err_unregister;
+ }
+
+ mtk_hsdma_hw_init(hsdma);
+
+ err = devm_request_irq(&pdev->dev, hsdma->irq,
+ mtk_hsdma_irq, 0,
+ dev_name(&pdev->dev), hsdma);
+ if (err) {
+ dev_err(&pdev->dev,
+ "request_irq failed with err %d\n", err);
+ goto err_free;
+ }
+
+ platform_set_drvdata(pdev, hsdma);
+
+ dev_info(&pdev->dev, "MediaTek HSDMA driver registered\n");
+
+ return 0;
+
+err_free:
+ mtk_hsdma_hw_deinit(hsdma);
+ of_dma_controller_free(pdev->dev.of_node);
+err_unregister:
+ dma_async_device_unregister(dd);
+
+ return err;
+}
+
+static int mtk_hsdma_remove(struct platform_device *pdev)
+{
+ struct mtk_hsdma_device *hsdma = platform_get_drvdata(pdev);
+ struct mtk_hsdma_vchan *vc;
+ int i;
+
+ /* Kill VC task */
+ for (i = 0; i < hsdma->dma_requests; i++) {
+ vc = &hsdma->vc[i];
+
+ list_del(&vc->vc.chan.device_node);
+ tasklet_kill(&vc->vc.task);
+ }
+
+ /* Disable DMA interrupt */
+ mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, 0);
+
+ /* Waits for any pending IRQ handlers to complete */
+ synchronize_irq(hsdma->irq);
+
+ /* Disable hardware */
+ mtk_hsdma_hw_deinit(hsdma);
+
+ dma_async_device_unregister(&hsdma->ddev);
+ of_dma_controller_free(pdev->dev.of_node);
+
+ return 0;
+}
+
+static struct platform_driver mtk_hsdma_driver = {
+ .probe = mtk_hsdma_probe,
+ .remove = mtk_hsdma_remove,
+ .driver = {
+ .name = KBUILD_MODNAME,
+ .of_match_table = mtk_hsdma_match,
+ },
+};
+module_platform_driver(mtk_hsdma_driver);
+
+MODULE_DESCRIPTION("MediaTek High-Speed DMA Controller Driver");
+MODULE_AUTHOR("Sean Wang <sean.wang@mediatek.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/dma/mediatek/mtk-uart-apdma.c b/drivers/dma/mediatek/mtk-uart-apdma.c
new file mode 100644
index 000000000..0acf6a92a
--- /dev/null
+++ b/drivers/dma/mediatek/mtk-uart-apdma.c
@@ -0,0 +1,656 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * MediaTek UART APDMA driver.
+ *
+ * Copyright (c) 2019 MediaTek Inc.
+ * Author: Long Cheng <long.cheng@mediatek.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/of_dma.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+#include "../virt-dma.h"
+
+/* The default number of virtual channel */
+#define MTK_UART_APDMA_NR_VCHANS 8
+
+#define VFF_EN_B BIT(0)
+#define VFF_STOP_B BIT(0)
+#define VFF_FLUSH_B BIT(0)
+#define VFF_4G_EN_B BIT(0)
+/* rx valid size >= vff thre */
+#define VFF_RX_INT_EN_B (BIT(0) | BIT(1))
+/* tx left size >= vff thre */
+#define VFF_TX_INT_EN_B BIT(0)
+#define VFF_WARM_RST_B BIT(0)
+#define VFF_RX_INT_CLR_B (BIT(0) | BIT(1))
+#define VFF_TX_INT_CLR_B 0
+#define VFF_STOP_CLR_B 0
+#define VFF_EN_CLR_B 0
+#define VFF_INT_EN_CLR_B 0
+#define VFF_4G_SUPPORT_CLR_B 0
+
+/*
+ * interrupt trigger level for tx
+ * if threshold is n, no polling is required to start tx.
+ * otherwise need polling VFF_FLUSH.
+ */
+#define VFF_TX_THRE(n) (n)
+/* interrupt trigger level for rx */
+#define VFF_RX_THRE(n) ((n) * 3 / 4)
+
+#define VFF_RING_SIZE 0xffff
+/* invert this bit when wrap ring head again */
+#define VFF_RING_WRAP 0x10000
+
+#define VFF_INT_FLAG 0x00
+#define VFF_INT_EN 0x04
+#define VFF_EN 0x08
+#define VFF_RST 0x0c
+#define VFF_STOP 0x10
+#define VFF_FLUSH 0x14
+#define VFF_ADDR 0x1c
+#define VFF_LEN 0x24
+#define VFF_THRE 0x28
+#define VFF_WPT 0x2c
+#define VFF_RPT 0x30
+/* TX: the buffer size HW can read. RX: the buffer size SW can read. */
+#define VFF_VALID_SIZE 0x3c
+/* TX: the buffer size SW can write. RX: the buffer size HW can write. */
+#define VFF_LEFT_SIZE 0x40
+#define VFF_DEBUG_STATUS 0x50
+#define VFF_4G_SUPPORT 0x54
+
+struct mtk_uart_apdmadev {
+ struct dma_device ddev;
+ struct clk *clk;
+ bool support_33bits;
+ unsigned int dma_requests;
+};
+
+struct mtk_uart_apdma_desc {
+ struct virt_dma_desc vd;
+
+ dma_addr_t addr;
+ unsigned int avail_len;
+};
+
+struct mtk_chan {
+ struct virt_dma_chan vc;
+ struct dma_slave_config cfg;
+ struct mtk_uart_apdma_desc *desc;
+ enum dma_transfer_direction dir;
+
+ void __iomem *base;
+ unsigned int irq;
+
+ unsigned int rx_status;
+};
+
+static inline struct mtk_uart_apdmadev *
+to_mtk_uart_apdma_dev(struct dma_device *d)
+{
+ return container_of(d, struct mtk_uart_apdmadev, ddev);
+}
+
+static inline struct mtk_chan *to_mtk_uart_apdma_chan(struct dma_chan *c)
+{
+ return container_of(c, struct mtk_chan, vc.chan);
+}
+
+static inline struct mtk_uart_apdma_desc *to_mtk_uart_apdma_desc
+ (struct dma_async_tx_descriptor *t)
+{
+ return container_of(t, struct mtk_uart_apdma_desc, vd.tx);
+}
+
+static void mtk_uart_apdma_write(struct mtk_chan *c,
+ unsigned int reg, unsigned int val)
+{
+ writel(val, c->base + reg);
+}
+
+static unsigned int mtk_uart_apdma_read(struct mtk_chan *c, unsigned int reg)
+{
+ return readl(c->base + reg);
+}
+
+static void mtk_uart_apdma_desc_free(struct virt_dma_desc *vd)
+{
+ kfree(container_of(vd, struct mtk_uart_apdma_desc, vd));
+}
+
+static void mtk_uart_apdma_start_tx(struct mtk_chan *c)
+{
+ struct mtk_uart_apdmadev *mtkd =
+ to_mtk_uart_apdma_dev(c->vc.chan.device);
+ struct mtk_uart_apdma_desc *d = c->desc;
+ unsigned int wpt, vff_sz;
+
+ vff_sz = c->cfg.dst_port_window_size;
+ if (!mtk_uart_apdma_read(c, VFF_LEN)) {
+ mtk_uart_apdma_write(c, VFF_ADDR, d->addr);
+ mtk_uart_apdma_write(c, VFF_LEN, vff_sz);
+ mtk_uart_apdma_write(c, VFF_THRE, VFF_TX_THRE(vff_sz));
+ mtk_uart_apdma_write(c, VFF_WPT, 0);
+ mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_TX_INT_CLR_B);
+
+ if (mtkd->support_33bits)
+ mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_EN_B);
+ }
+
+ mtk_uart_apdma_write(c, VFF_EN, VFF_EN_B);
+ if (mtk_uart_apdma_read(c, VFF_EN) != VFF_EN_B)
+ dev_err(c->vc.chan.device->dev, "Enable TX fail\n");
+
+ if (!mtk_uart_apdma_read(c, VFF_LEFT_SIZE)) {
+ mtk_uart_apdma_write(c, VFF_INT_EN, VFF_TX_INT_EN_B);
+ return;
+ }
+
+ wpt = mtk_uart_apdma_read(c, VFF_WPT);
+
+ wpt += c->desc->avail_len;
+ if ((wpt & VFF_RING_SIZE) == vff_sz)
+ wpt = (wpt & VFF_RING_WRAP) ^ VFF_RING_WRAP;
+
+ /* Let DMA start moving data */
+ mtk_uart_apdma_write(c, VFF_WPT, wpt);
+
+ /* HW auto set to 0 when left size >= threshold */
+ mtk_uart_apdma_write(c, VFF_INT_EN, VFF_TX_INT_EN_B);
+ if (!mtk_uart_apdma_read(c, VFF_FLUSH))
+ mtk_uart_apdma_write(c, VFF_FLUSH, VFF_FLUSH_B);
+}
+
+static void mtk_uart_apdma_start_rx(struct mtk_chan *c)
+{
+ struct mtk_uart_apdmadev *mtkd =
+ to_mtk_uart_apdma_dev(c->vc.chan.device);
+ struct mtk_uart_apdma_desc *d = c->desc;
+ unsigned int vff_sz;
+
+ vff_sz = c->cfg.src_port_window_size;
+ if (!mtk_uart_apdma_read(c, VFF_LEN)) {
+ mtk_uart_apdma_write(c, VFF_ADDR, d->addr);
+ mtk_uart_apdma_write(c, VFF_LEN, vff_sz);
+ mtk_uart_apdma_write(c, VFF_THRE, VFF_RX_THRE(vff_sz));
+ mtk_uart_apdma_write(c, VFF_RPT, 0);
+ mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_RX_INT_CLR_B);
+
+ if (mtkd->support_33bits)
+ mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_EN_B);
+ }
+
+ mtk_uart_apdma_write(c, VFF_INT_EN, VFF_RX_INT_EN_B);
+ mtk_uart_apdma_write(c, VFF_EN, VFF_EN_B);
+ if (mtk_uart_apdma_read(c, VFF_EN) != VFF_EN_B)
+ dev_err(c->vc.chan.device->dev, "Enable RX fail\n");
+}
+
+static void mtk_uart_apdma_tx_handler(struct mtk_chan *c)
+{
+ mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_TX_INT_CLR_B);
+ mtk_uart_apdma_write(c, VFF_INT_EN, VFF_INT_EN_CLR_B);
+ mtk_uart_apdma_write(c, VFF_EN, VFF_EN_CLR_B);
+}
+
+static void mtk_uart_apdma_rx_handler(struct mtk_chan *c)
+{
+ struct mtk_uart_apdma_desc *d = c->desc;
+ unsigned int len, wg, rg;
+ int cnt;
+
+ mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_RX_INT_CLR_B);
+
+ if (!mtk_uart_apdma_read(c, VFF_VALID_SIZE))
+ return;
+
+ mtk_uart_apdma_write(c, VFF_EN, VFF_EN_CLR_B);
+ mtk_uart_apdma_write(c, VFF_INT_EN, VFF_INT_EN_CLR_B);
+
+ len = c->cfg.src_port_window_size;
+ rg = mtk_uart_apdma_read(c, VFF_RPT);
+ wg = mtk_uart_apdma_read(c, VFF_WPT);
+ cnt = (wg & VFF_RING_SIZE) - (rg & VFF_RING_SIZE);
+
+ /*
+ * The buffer is ring buffer. If wrap bit different,
+ * represents the start of the next cycle for WPT
+ */
+ if ((rg ^ wg) & VFF_RING_WRAP)
+ cnt += len;
+
+ c->rx_status = d->avail_len - cnt;
+ mtk_uart_apdma_write(c, VFF_RPT, wg);
+}
+
+static void mtk_uart_apdma_chan_complete_handler(struct mtk_chan *c)
+{
+ struct mtk_uart_apdma_desc *d = c->desc;
+
+ if (d) {
+ list_del(&d->vd.node);
+ vchan_cookie_complete(&d->vd);
+ c->desc = NULL;
+ }
+}
+
+static irqreturn_t mtk_uart_apdma_irq_handler(int irq, void *dev_id)
+{
+ struct dma_chan *chan = (struct dma_chan *)dev_id;
+ struct mtk_chan *c = to_mtk_uart_apdma_chan(chan);
+ unsigned long flags;
+
+ spin_lock_irqsave(&c->vc.lock, flags);
+ if (c->dir == DMA_DEV_TO_MEM)
+ mtk_uart_apdma_rx_handler(c);
+ else if (c->dir == DMA_MEM_TO_DEV)
+ mtk_uart_apdma_tx_handler(c);
+ mtk_uart_apdma_chan_complete_handler(c);
+ spin_unlock_irqrestore(&c->vc.lock, flags);
+
+ return IRQ_HANDLED;
+}
+
+static int mtk_uart_apdma_alloc_chan_resources(struct dma_chan *chan)
+{
+ struct mtk_uart_apdmadev *mtkd = to_mtk_uart_apdma_dev(chan->device);
+ struct mtk_chan *c = to_mtk_uart_apdma_chan(chan);
+ unsigned int status;
+ int ret;
+
+ ret = pm_runtime_resume_and_get(mtkd->ddev.dev);
+ if (ret < 0) {
+ pm_runtime_put_noidle(chan->device->dev);
+ return ret;
+ }
+
+ mtk_uart_apdma_write(c, VFF_ADDR, 0);
+ mtk_uart_apdma_write(c, VFF_THRE, 0);
+ mtk_uart_apdma_write(c, VFF_LEN, 0);
+ mtk_uart_apdma_write(c, VFF_RST, VFF_WARM_RST_B);
+
+ ret = readx_poll_timeout(readl, c->base + VFF_EN,
+ status, !status, 10, 100);
+ if (ret)
+ goto err_pm;
+
+ ret = request_irq(c->irq, mtk_uart_apdma_irq_handler,
+ IRQF_TRIGGER_NONE, KBUILD_MODNAME, chan);
+ if (ret < 0) {
+ dev_err(chan->device->dev, "Can't request dma IRQ\n");
+ ret = -EINVAL;
+ goto err_pm;
+ }
+
+ if (mtkd->support_33bits)
+ mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_SUPPORT_CLR_B);
+
+err_pm:
+ pm_runtime_put_noidle(mtkd->ddev.dev);
+ return ret;
+}
+
+static void mtk_uart_apdma_free_chan_resources(struct dma_chan *chan)
+{
+ struct mtk_uart_apdmadev *mtkd = to_mtk_uart_apdma_dev(chan->device);
+ struct mtk_chan *c = to_mtk_uart_apdma_chan(chan);
+
+ free_irq(c->irq, chan);
+
+ tasklet_kill(&c->vc.task);
+
+ vchan_free_chan_resources(&c->vc);
+
+ pm_runtime_put_sync(mtkd->ddev.dev);
+}
+
+static enum dma_status mtk_uart_apdma_tx_status(struct dma_chan *chan,
+ dma_cookie_t cookie,
+ struct dma_tx_state *txstate)
+{
+ struct mtk_chan *c = to_mtk_uart_apdma_chan(chan);
+ enum dma_status ret;
+
+ ret = dma_cookie_status(chan, cookie, txstate);
+ if (!txstate)
+ return ret;
+
+ dma_set_residue(txstate, c->rx_status);
+
+ return ret;
+}
+
+/*
+ * dmaengine_prep_slave_single will call the function. and sglen is 1.
+ * 8250 uart using one ring buffer, and deal with one sg.
+ */
+static struct dma_async_tx_descriptor *mtk_uart_apdma_prep_slave_sg
+ (struct dma_chan *chan, struct scatterlist *sgl,
+ unsigned int sglen, enum dma_transfer_direction dir,
+ unsigned long tx_flags, void *context)
+{
+ struct mtk_chan *c = to_mtk_uart_apdma_chan(chan);
+ struct mtk_uart_apdma_desc *d;
+
+ if (!is_slave_direction(dir) || sglen != 1)
+ return NULL;
+
+ /* Now allocate and setup the descriptor */
+ d = kzalloc(sizeof(*d), GFP_NOWAIT);
+ if (!d)
+ return NULL;
+
+ d->avail_len = sg_dma_len(sgl);
+ d->addr = sg_dma_address(sgl);
+ c->dir = dir;
+
+ return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
+}
+
+static void mtk_uart_apdma_issue_pending(struct dma_chan *chan)
+{
+ struct mtk_chan *c = to_mtk_uart_apdma_chan(chan);
+ struct virt_dma_desc *vd;
+ unsigned long flags;
+
+ spin_lock_irqsave(&c->vc.lock, flags);
+ if (vchan_issue_pending(&c->vc) && !c->desc) {
+ vd = vchan_next_desc(&c->vc);
+ c->desc = to_mtk_uart_apdma_desc(&vd->tx);
+
+ if (c->dir == DMA_DEV_TO_MEM)
+ mtk_uart_apdma_start_rx(c);
+ else if (c->dir == DMA_MEM_TO_DEV)
+ mtk_uart_apdma_start_tx(c);
+ }
+
+ spin_unlock_irqrestore(&c->vc.lock, flags);
+}
+
+static int mtk_uart_apdma_slave_config(struct dma_chan *chan,
+ struct dma_slave_config *config)
+{
+ struct mtk_chan *c = to_mtk_uart_apdma_chan(chan);
+
+ memcpy(&c->cfg, config, sizeof(*config));
+
+ return 0;
+}
+
+static int mtk_uart_apdma_terminate_all(struct dma_chan *chan)
+{
+ struct mtk_chan *c = to_mtk_uart_apdma_chan(chan);
+ unsigned long flags;
+ unsigned int status;
+ LIST_HEAD(head);
+ int ret;
+
+ mtk_uart_apdma_write(c, VFF_FLUSH, VFF_FLUSH_B);
+
+ ret = readx_poll_timeout(readl, c->base + VFF_FLUSH,
+ status, status != VFF_FLUSH_B, 10, 100);
+ if (ret)
+ dev_err(c->vc.chan.device->dev, "flush: fail, status=0x%x\n",
+ mtk_uart_apdma_read(c, VFF_DEBUG_STATUS));
+
+ /*
+ * Stop need 3 steps.
+ * 1. set stop to 1
+ * 2. wait en to 0
+ * 3. set stop as 0
+ */
+ mtk_uart_apdma_write(c, VFF_STOP, VFF_STOP_B);
+ ret = readx_poll_timeout(readl, c->base + VFF_EN,
+ status, !status, 10, 100);
+ if (ret)
+ dev_err(c->vc.chan.device->dev, "stop: fail, status=0x%x\n",
+ mtk_uart_apdma_read(c, VFF_DEBUG_STATUS));
+
+ mtk_uart_apdma_write(c, VFF_STOP, VFF_STOP_CLR_B);
+ mtk_uart_apdma_write(c, VFF_INT_EN, VFF_INT_EN_CLR_B);
+
+ if (c->dir == DMA_DEV_TO_MEM)
+ mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_RX_INT_CLR_B);
+ else if (c->dir == DMA_MEM_TO_DEV)
+ mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_TX_INT_CLR_B);
+
+ synchronize_irq(c->irq);
+
+ spin_lock_irqsave(&c->vc.lock, flags);
+ vchan_get_all_descriptors(&c->vc, &head);
+ spin_unlock_irqrestore(&c->vc.lock, flags);
+
+ vchan_dma_desc_free_list(&c->vc, &head);
+
+ return 0;
+}
+
+static int mtk_uart_apdma_device_pause(struct dma_chan *chan)
+{
+ struct mtk_chan *c = to_mtk_uart_apdma_chan(chan);
+ unsigned long flags;
+
+ spin_lock_irqsave(&c->vc.lock, flags);
+
+ mtk_uart_apdma_write(c, VFF_EN, VFF_EN_CLR_B);
+ mtk_uart_apdma_write(c, VFF_INT_EN, VFF_INT_EN_CLR_B);
+
+ spin_unlock_irqrestore(&c->vc.lock, flags);
+ synchronize_irq(c->irq);
+
+ return 0;
+}
+
+static void mtk_uart_apdma_free(struct mtk_uart_apdmadev *mtkd)
+{
+ while (!list_empty(&mtkd->ddev.channels)) {
+ struct mtk_chan *c = list_first_entry(&mtkd->ddev.channels,
+ struct mtk_chan, vc.chan.device_node);
+
+ list_del(&c->vc.chan.device_node);
+ tasklet_kill(&c->vc.task);
+ }
+}
+
+static const struct of_device_id mtk_uart_apdma_match[] = {
+ { .compatible = "mediatek,mt6577-uart-dma", },
+ { /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, mtk_uart_apdma_match);
+
+static int mtk_uart_apdma_probe(struct platform_device *pdev)
+{
+ struct device_node *np = pdev->dev.of_node;
+ struct mtk_uart_apdmadev *mtkd;
+ int bit_mask = 32, rc;
+ struct mtk_chan *c;
+ unsigned int i;
+
+ mtkd = devm_kzalloc(&pdev->dev, sizeof(*mtkd), GFP_KERNEL);
+ if (!mtkd)
+ return -ENOMEM;
+
+ mtkd->clk = devm_clk_get(&pdev->dev, NULL);
+ if (IS_ERR(mtkd->clk)) {
+ dev_err(&pdev->dev, "No clock specified\n");
+ rc = PTR_ERR(mtkd->clk);
+ return rc;
+ }
+
+ if (of_property_read_bool(np, "mediatek,dma-33bits"))
+ mtkd->support_33bits = true;
+
+ if (mtkd->support_33bits)
+ bit_mask = 33;
+
+ rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(bit_mask));
+ if (rc)
+ return rc;
+
+ dma_cap_set(DMA_SLAVE, mtkd->ddev.cap_mask);
+ mtkd->ddev.device_alloc_chan_resources =
+ mtk_uart_apdma_alloc_chan_resources;
+ mtkd->ddev.device_free_chan_resources =
+ mtk_uart_apdma_free_chan_resources;
+ mtkd->ddev.device_tx_status = mtk_uart_apdma_tx_status;
+ mtkd->ddev.device_issue_pending = mtk_uart_apdma_issue_pending;
+ mtkd->ddev.device_prep_slave_sg = mtk_uart_apdma_prep_slave_sg;
+ mtkd->ddev.device_config = mtk_uart_apdma_slave_config;
+ mtkd->ddev.device_pause = mtk_uart_apdma_device_pause;
+ mtkd->ddev.device_terminate_all = mtk_uart_apdma_terminate_all;
+ mtkd->ddev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE);
+ mtkd->ddev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE);
+ mtkd->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
+ mtkd->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
+ mtkd->ddev.dev = &pdev->dev;
+ INIT_LIST_HEAD(&mtkd->ddev.channels);
+
+ mtkd->dma_requests = MTK_UART_APDMA_NR_VCHANS;
+ if (of_property_read_u32(np, "dma-requests", &mtkd->dma_requests)) {
+ dev_info(&pdev->dev,
+ "Using %u as missing dma-requests property\n",
+ MTK_UART_APDMA_NR_VCHANS);
+ }
+
+ for (i = 0; i < mtkd->dma_requests; i++) {
+ c = devm_kzalloc(mtkd->ddev.dev, sizeof(*c), GFP_KERNEL);
+ if (!c) {
+ rc = -ENODEV;
+ goto err_no_dma;
+ }
+
+ c->base = devm_platform_ioremap_resource(pdev, i);
+ if (IS_ERR(c->base)) {
+ rc = PTR_ERR(c->base);
+ goto err_no_dma;
+ }
+ c->vc.desc_free = mtk_uart_apdma_desc_free;
+ vchan_init(&c->vc, &mtkd->ddev);
+
+ rc = platform_get_irq(pdev, i);
+ if (rc < 0)
+ goto err_no_dma;
+ c->irq = rc;
+ }
+
+ pm_runtime_enable(&pdev->dev);
+ pm_runtime_set_active(&pdev->dev);
+
+ rc = dma_async_device_register(&mtkd->ddev);
+ if (rc)
+ goto rpm_disable;
+
+ platform_set_drvdata(pdev, mtkd);
+
+ /* Device-tree DMA controller registration */
+ rc = of_dma_controller_register(np, of_dma_xlate_by_chan_id, mtkd);
+ if (rc)
+ goto dma_remove;
+
+ return rc;
+
+dma_remove:
+ dma_async_device_unregister(&mtkd->ddev);
+rpm_disable:
+ pm_runtime_disable(&pdev->dev);
+err_no_dma:
+ mtk_uart_apdma_free(mtkd);
+ return rc;
+}
+
+static int mtk_uart_apdma_remove(struct platform_device *pdev)
+{
+ struct mtk_uart_apdmadev *mtkd = platform_get_drvdata(pdev);
+
+ of_dma_controller_free(pdev->dev.of_node);
+
+ mtk_uart_apdma_free(mtkd);
+
+ dma_async_device_unregister(&mtkd->ddev);
+
+ pm_runtime_disable(&pdev->dev);
+
+ return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int mtk_uart_apdma_suspend(struct device *dev)
+{
+ struct mtk_uart_apdmadev *mtkd = dev_get_drvdata(dev);
+
+ if (!pm_runtime_suspended(dev))
+ clk_disable_unprepare(mtkd->clk);
+
+ return 0;
+}
+
+static int mtk_uart_apdma_resume(struct device *dev)
+{
+ int ret;
+ struct mtk_uart_apdmadev *mtkd = dev_get_drvdata(dev);
+
+ if (!pm_runtime_suspended(dev)) {
+ ret = clk_prepare_enable(mtkd->clk);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+#endif /* CONFIG_PM_SLEEP */
+
+#ifdef CONFIG_PM
+static int mtk_uart_apdma_runtime_suspend(struct device *dev)
+{
+ struct mtk_uart_apdmadev *mtkd = dev_get_drvdata(dev);
+
+ clk_disable_unprepare(mtkd->clk);
+
+ return 0;
+}
+
+static int mtk_uart_apdma_runtime_resume(struct device *dev)
+{
+ struct mtk_uart_apdmadev *mtkd = dev_get_drvdata(dev);
+
+ return clk_prepare_enable(mtkd->clk);
+}
+#endif /* CONFIG_PM */
+
+static const struct dev_pm_ops mtk_uart_apdma_pm_ops = {
+ SET_SYSTEM_SLEEP_PM_OPS(mtk_uart_apdma_suspend, mtk_uart_apdma_resume)
+ SET_RUNTIME_PM_OPS(mtk_uart_apdma_runtime_suspend,
+ mtk_uart_apdma_runtime_resume, NULL)
+};
+
+static struct platform_driver mtk_uart_apdma_driver = {
+ .probe = mtk_uart_apdma_probe,
+ .remove = mtk_uart_apdma_remove,
+ .driver = {
+ .name = KBUILD_MODNAME,
+ .pm = &mtk_uart_apdma_pm_ops,
+ .of_match_table = of_match_ptr(mtk_uart_apdma_match),
+ },
+};
+
+module_platform_driver(mtk_uart_apdma_driver);
+
+MODULE_DESCRIPTION("MediaTek UART APDMA Controller Driver");
+MODULE_AUTHOR("Long Cheng <long.cheng@mediatek.com>");
+MODULE_LICENSE("GPL v2");