summaryrefslogtreecommitdiffstats
path: root/drivers/usb/gadget/udc/cdns2
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/usb/gadget/udc/cdns2')
-rw-r--r--drivers/usb/gadget/udc/cdns2/Kconfig11
-rw-r--r--drivers/usb/gadget/udc/cdns2/Makefile7
-rw-r--r--drivers/usb/gadget/udc/cdns2/cdns2-debug.h203
-rw-r--r--drivers/usb/gadget/udc/cdns2/cdns2-ep0.c659
-rw-r--r--drivers/usb/gadget/udc/cdns2/cdns2-gadget.c2474
-rw-r--r--drivers/usb/gadget/udc/cdns2/cdns2-gadget.h707
-rw-r--r--drivers/usb/gadget/udc/cdns2/cdns2-pci.c138
-rw-r--r--drivers/usb/gadget/udc/cdns2/cdns2-trace.c11
-rw-r--r--drivers/usb/gadget/udc/cdns2/cdns2-trace.h605
9 files changed, 4815 insertions, 0 deletions
diff --git a/drivers/usb/gadget/udc/cdns2/Kconfig b/drivers/usb/gadget/udc/cdns2/Kconfig
new file mode 100644
index 0000000000..c07d353903
--- /dev/null
+++ b/drivers/usb/gadget/udc/cdns2/Kconfig
@@ -0,0 +1,11 @@
+config USB_CDNS2_UDC
+ tristate "Cadence USBHS Device Controller"
+ depends on USB_PCI && ACPI && HAS_DMA
+ help
+ Cadence USBHS Device controller is a PCI based USB peripheral
+ controller which supports both full and high speed USB 2.0
+ data transfers.
+
+ Say "y" to link the driver statically, or "m" to build a
+ dynamically linked module called "cdns2-udc-pci.ko" and to
+ force all gadget drivers to also be dynamically linked.
diff --git a/drivers/usb/gadget/udc/cdns2/Makefile b/drivers/usb/gadget/udc/cdns2/Makefile
new file mode 100644
index 0000000000..a1ffbbe2e7
--- /dev/null
+++ b/drivers/usb/gadget/udc/cdns2/Makefile
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0
+# define_trace.h needs to know how to find our header
+CFLAGS_cdns2-trace.o := -I$(src)
+
+obj-$(CONFIG_USB_CDNS2_UDC) += cdns2-udc-pci.o
+cdns2-udc-pci-$(CONFIG_USB_CDNS2_UDC) += cdns2-pci.o cdns2-gadget.o cdns2-ep0.o
+cdns2-udc-pci-$(CONFIG_TRACING) += cdns2-trace.o
diff --git a/drivers/usb/gadget/udc/cdns2/cdns2-debug.h b/drivers/usb/gadget/udc/cdns2/cdns2-debug.h
new file mode 100644
index 0000000000..be9ae0d28a
--- /dev/null
+++ b/drivers/usb/gadget/udc/cdns2/cdns2-debug.h
@@ -0,0 +1,203 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Cadence USBHS-DEV Driver.
+ * Debug header file.
+ *
+ * Copyright (C) 2023 Cadence.
+ *
+ * Author: Pawel Laszczak <pawell@cadence.com>
+ */
+
+#ifndef __LINUX_CDNS2_DEBUG
+#define __LINUX_CDNS2_DEBUG
+
+static inline const char *cdns2_decode_usb_irq(char *str, size_t size,
+ u8 usb_irq, u8 ext_irq)
+{
+ int ret;
+
+ ret = snprintf(str, size, "usbirq: 0x%02x - ", usb_irq);
+
+ if (usb_irq & USBIRQ_SOF)
+ ret += snprintf(str + ret, size - ret, "SOF ");
+ if (usb_irq & USBIRQ_SUTOK)
+ ret += snprintf(str + ret, size - ret, "SUTOK ");
+ if (usb_irq & USBIRQ_SUDAV)
+ ret += snprintf(str + ret, size - ret, "SETUP ");
+ if (usb_irq & USBIRQ_SUSPEND)
+ ret += snprintf(str + ret, size - ret, "Suspend ");
+ if (usb_irq & USBIRQ_URESET)
+ ret += snprintf(str + ret, size - ret, "Reset ");
+ if (usb_irq & USBIRQ_HSPEED)
+ ret += snprintf(str + ret, size - ret, "HS ");
+ if (usb_irq & USBIRQ_LPM)
+ ret += snprintf(str + ret, size - ret, "LPM ");
+
+ ret += snprintf(str + ret, size - ret, ", EXT: 0x%02x - ", ext_irq);
+
+ if (ext_irq & EXTIRQ_WAKEUP)
+ ret += snprintf(str + ret, size - ret, "Wakeup ");
+ if (ext_irq & EXTIRQ_VBUSFAULT_FALL)
+ ret += snprintf(str + ret, size - ret, "VBUS_FALL ");
+ if (ext_irq & EXTIRQ_VBUSFAULT_RISE)
+ ret += snprintf(str + ret, size - ret, "VBUS_RISE ");
+
+ if (ret >= size)
+ pr_info("CDNS2: buffer overflowed.\n");
+
+ return str;
+}
+
+static inline const char *cdns2_decode_dma_irq(char *str, size_t size,
+ u32 ep_ists, u32 ep_sts,
+ const char *ep_name)
+{
+ int ret;
+
+ ret = snprintf(str, size, "ISTS: %08x, %s: %08x ",
+ ep_ists, ep_name, ep_sts);
+
+ if (ep_sts & DMA_EP_STS_IOC)
+ ret += snprintf(str + ret, size - ret, "IOC ");
+ if (ep_sts & DMA_EP_STS_ISP)
+ ret += snprintf(str + ret, size - ret, "ISP ");
+ if (ep_sts & DMA_EP_STS_DESCMIS)
+ ret += snprintf(str + ret, size - ret, "DESCMIS ");
+ if (ep_sts & DMA_EP_STS_TRBERR)
+ ret += snprintf(str + ret, size - ret, "TRBERR ");
+ if (ep_sts & DMA_EP_STS_OUTSMM)
+ ret += snprintf(str + ret, size - ret, "OUTSMM ");
+ if (ep_sts & DMA_EP_STS_ISOERR)
+ ret += snprintf(str + ret, size - ret, "ISOERR ");
+ if (ep_sts & DMA_EP_STS_DBUSY)
+ ret += snprintf(str + ret, size - ret, "DBUSY ");
+ if (DMA_EP_STS_CCS(ep_sts))
+ ret += snprintf(str + ret, size - ret, "CCS ");
+
+ if (ret >= size)
+ pr_info("CDNS2: buffer overflowed.\n");
+
+ return str;
+}
+
+static inline const char *cdns2_decode_epx_irq(char *str, size_t size,
+ char *ep_name, u32 ep_ists,
+ u32 ep_sts)
+{
+ return cdns2_decode_dma_irq(str, size, ep_ists, ep_sts, ep_name);
+}
+
+static inline const char *cdns2_decode_ep0_irq(char *str, size_t size,
+ u32 ep_ists, u32 ep_sts,
+ int dir)
+{
+ return cdns2_decode_dma_irq(str, size, ep_ists, ep_sts,
+ dir ? "ep0IN" : "ep0OUT");
+}
+
+static inline const char *cdns2_raw_ring(struct cdns2_endpoint *pep,
+ struct cdns2_trb *trbs,
+ char *str, size_t size)
+{
+ struct cdns2_ring *ring = &pep->ring;
+ struct cdns2_trb *trb;
+ dma_addr_t dma;
+ int ret;
+ int i;
+
+ ret = snprintf(str, size, "\n\t\tTR for %s:", pep->name);
+
+ trb = &trbs[ring->dequeue];
+ dma = cdns2_trb_virt_to_dma(pep, trb);
+ ret += snprintf(str + ret, size - ret,
+ "\n\t\tRing deq index: %d, trb: V=%p, P=0x%pad\n",
+ ring->dequeue, trb, &dma);
+
+ trb = &trbs[ring->enqueue];
+ dma = cdns2_trb_virt_to_dma(pep, trb);
+ ret += snprintf(str + ret, size - ret,
+ "\t\tRing enq index: %d, trb: V=%p, P=0x%pad\n",
+ ring->enqueue, trb, &dma);
+
+ ret += snprintf(str + ret, size - ret,
+ "\t\tfree trbs: %d, CCS=%d, PCS=%d\n",
+ ring->free_trbs, ring->ccs, ring->pcs);
+
+ if (TRBS_PER_SEGMENT > 40) {
+ ret += snprintf(str + ret, size - ret,
+ "\t\tTransfer ring %d too big\n", TRBS_PER_SEGMENT);
+ return str;
+ }
+
+ dma = ring->dma;
+ for (i = 0; i < TRBS_PER_SEGMENT; ++i) {
+ trb = &trbs[i];
+ ret += snprintf(str + ret, size - ret,
+ "\t\t@%pad %08x %08x %08x\n", &dma,
+ le32_to_cpu(trb->buffer),
+ le32_to_cpu(trb->length),
+ le32_to_cpu(trb->control));
+ dma += sizeof(*trb);
+ }
+
+ if (ret >= size)
+ pr_info("CDNS2: buffer overflowed.\n");
+
+ return str;
+}
+
+static inline const char *cdns2_trb_type_string(u8 type)
+{
+ switch (type) {
+ case TRB_NORMAL:
+ return "Normal";
+ case TRB_LINK:
+ return "Link";
+ default:
+ return "UNKNOWN";
+ }
+}
+
+static inline const char *cdns2_decode_trb(char *str, size_t size, u32 flags,
+ u32 length, u32 buffer)
+{
+ int type = TRB_FIELD_TO_TYPE(flags);
+ int ret;
+
+ switch (type) {
+ case TRB_LINK:
+ ret = snprintf(str, size,
+ "LINK %08x type '%s' flags %c:%c:%c%c:%c",
+ buffer, cdns2_trb_type_string(type),
+ flags & TRB_CYCLE ? 'C' : 'c',
+ flags & TRB_TOGGLE ? 'T' : 't',
+ flags & TRB_CHAIN ? 'C' : 'c',
+ flags & TRB_CHAIN ? 'H' : 'h',
+ flags & TRB_IOC ? 'I' : 'i');
+ break;
+ case TRB_NORMAL:
+ ret = snprintf(str, size,
+ "type: '%s', Buffer: %08x, length: %ld, burst len: %ld, "
+ "flags %c:%c:%c%c:%c",
+ cdns2_trb_type_string(type),
+ buffer, TRB_LEN(length),
+ TRB_FIELD_TO_BURST(length),
+ flags & TRB_CYCLE ? 'C' : 'c',
+ flags & TRB_ISP ? 'I' : 'i',
+ flags & TRB_CHAIN ? 'C' : 'c',
+ flags & TRB_CHAIN ? 'H' : 'h',
+ flags & TRB_IOC ? 'I' : 'i');
+ break;
+ default:
+ ret = snprintf(str, size, "type '%s' -> raw %08x %08x %08x",
+ cdns2_trb_type_string(type),
+ buffer, length, flags);
+ }
+
+ if (ret >= size)
+ pr_info("CDNS2: buffer overflowed.\n");
+
+ return str;
+}
+
+#endif /*__LINUX_CDNS2_DEBUG*/
diff --git a/drivers/usb/gadget/udc/cdns2/cdns2-ep0.c b/drivers/usb/gadget/udc/cdns2/cdns2-ep0.c
new file mode 100644
index 0000000000..fa12a5d46f
--- /dev/null
+++ b/drivers/usb/gadget/udc/cdns2/cdns2-ep0.c
@@ -0,0 +1,659 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Cadence USBHS-DEV driver.
+ *
+ * Copyright (C) 2023 Cadence Design Systems.
+ *
+ * Authors: Pawel Laszczak <pawell@cadence.com>
+ */
+
+#include <linux/usb/composite.h>
+#include <asm/unaligned.h>
+
+#include "cdns2-gadget.h"
+#include "cdns2-trace.h"
+
+static struct usb_endpoint_descriptor cdns2_gadget_ep0_desc = {
+ .bLength = USB_DT_ENDPOINT_SIZE,
+ .bDescriptorType = USB_DT_ENDPOINT,
+ .bmAttributes = USB_ENDPOINT_XFER_CONTROL,
+ .wMaxPacketSize = cpu_to_le16(64)
+};
+
+static int cdns2_w_index_to_ep_index(u16 wIndex)
+{
+ if (!(wIndex & USB_ENDPOINT_NUMBER_MASK))
+ return 0;
+
+ return ((wIndex & USB_ENDPOINT_NUMBER_MASK) * 2) +
+ (wIndex & USB_ENDPOINT_DIR_MASK ? 1 : 0) - 1;
+}
+
+static bool cdns2_check_new_setup(struct cdns2_device *pdev)
+{
+ u8 reg;
+
+ reg = readb(&pdev->ep0_regs->cs);
+
+ return !!(reg & EP0CS_CHGSET);
+}
+
+static void cdns2_ep0_enqueue(struct cdns2_device *pdev, dma_addr_t dma_addr,
+ unsigned int length, int zlp)
+{
+ struct cdns2_adma_regs __iomem *regs = pdev->adma_regs;
+ struct cdns2_endpoint *pep = &pdev->eps[0];
+ struct cdns2_ring *ring = &pep->ring;
+
+ ring->trbs[0].buffer = cpu_to_le32(TRB_BUFFER(dma_addr));
+ ring->trbs[0].length = cpu_to_le32(TRB_LEN(length));
+
+ if (zlp) {
+ ring->trbs[0].control = cpu_to_le32(TRB_CYCLE |
+ TRB_TYPE(TRB_NORMAL));
+ ring->trbs[1].buffer = cpu_to_le32(TRB_BUFFER(dma_addr));
+ ring->trbs[1].length = cpu_to_le32(TRB_LEN(0));
+ ring->trbs[1].control = cpu_to_le32(TRB_CYCLE | TRB_IOC |
+ TRB_TYPE(TRB_NORMAL));
+ } else {
+ ring->trbs[0].control = cpu_to_le32(TRB_CYCLE | TRB_IOC |
+ TRB_TYPE(TRB_NORMAL));
+ ring->trbs[1].control = 0;
+ }
+
+ trace_cdns2_queue_trb(pep, ring->trbs);
+
+ if (!pep->dir)
+ writel(0, &pdev->ep0_regs->rxbc);
+
+ cdns2_select_ep(pdev, pep->dir);
+
+ writel(DMA_EP_STS_TRBERR, &regs->ep_sts);
+ writel(pep->ring.dma, &regs->ep_traddr);
+
+ trace_cdns2_doorbell_ep0(pep, readl(&regs->ep_traddr));
+
+ writel(DMA_EP_CMD_DRDY, &regs->ep_cmd);
+}
+
+static int cdns2_ep0_delegate_req(struct cdns2_device *pdev)
+{
+ int ret;
+
+ spin_unlock(&pdev->lock);
+ ret = pdev->gadget_driver->setup(&pdev->gadget, &pdev->setup);
+ spin_lock(&pdev->lock);
+
+ return ret;
+}
+
+static void cdns2_ep0_stall(struct cdns2_device *pdev)
+{
+ struct cdns2_endpoint *pep = &pdev->eps[0];
+ struct cdns2_request *preq;
+
+ preq = cdns2_next_preq(&pep->pending_list);
+ set_reg_bit_8(&pdev->ep0_regs->cs, EP0CS_DSTALL);
+
+ if (pdev->ep0_stage == CDNS2_DATA_STAGE && preq)
+ cdns2_gadget_giveback(pep, preq, -ECONNRESET);
+ else if (preq)
+ list_del_init(&preq->list);
+
+ pdev->ep0_stage = CDNS2_SETUP_STAGE;
+ pep->ep_state |= EP_STALLED;
+}
+
+static void cdns2_status_stage(struct cdns2_device *pdev)
+{
+ struct cdns2_endpoint *pep = &pdev->eps[0];
+ struct cdns2_request *preq;
+
+ preq = cdns2_next_preq(&pep->pending_list);
+ if (preq)
+ list_del_init(&preq->list);
+
+ pdev->ep0_stage = CDNS2_SETUP_STAGE;
+ writeb(EP0CS_HSNAK, &pdev->ep0_regs->cs);
+}
+
+static int cdns2_req_ep0_set_configuration(struct cdns2_device *pdev,
+ struct usb_ctrlrequest *ctrl_req)
+{
+ enum usb_device_state state = pdev->gadget.state;
+ u32 config = le16_to_cpu(ctrl_req->wValue);
+ int ret;
+
+ if (state < USB_STATE_ADDRESS) {
+ dev_err(pdev->dev, "Set Configuration - bad device state\n");
+ return -EINVAL;
+ }
+
+ ret = cdns2_ep0_delegate_req(pdev);
+ if (ret)
+ return ret;
+
+ trace_cdns2_device_state(config ? "configured" : "addressed");
+
+ if (!config)
+ usb_gadget_set_state(&pdev->gadget, USB_STATE_ADDRESS);
+
+ return 0;
+}
+
+static int cdns2_req_ep0_set_address(struct cdns2_device *pdev, u32 addr)
+{
+ enum usb_device_state device_state = pdev->gadget.state;
+ u8 reg;
+
+ if (addr > USB_DEVICE_MAX_ADDRESS) {
+ dev_err(pdev->dev,
+ "Device address (%d) cannot be greater than %d\n",
+ addr, USB_DEVICE_MAX_ADDRESS);
+ return -EINVAL;
+ }
+
+ if (device_state == USB_STATE_CONFIGURED) {
+ dev_err(pdev->dev,
+ "can't set_address from configured state\n");
+ return -EINVAL;
+ }
+
+ reg = readb(&pdev->usb_regs->fnaddr);
+ pdev->dev_address = reg;
+
+ usb_gadget_set_state(&pdev->gadget,
+ (addr ? USB_STATE_ADDRESS : USB_STATE_DEFAULT));
+
+ trace_cdns2_device_state(addr ? "addressed" : "default");
+
+ return 0;
+}
+
+static int cdns2_req_ep0_handle_status(struct cdns2_device *pdev,
+ struct usb_ctrlrequest *ctrl)
+{
+ struct cdns2_endpoint *pep;
+ __le16 *response_pkt;
+ u16 status = 0;
+ int ep_sts;
+ u32 recip;
+
+ recip = ctrl->bRequestType & USB_RECIP_MASK;
+
+ switch (recip) {
+ case USB_RECIP_DEVICE:
+ status = pdev->gadget.is_selfpowered;
+ status |= pdev->may_wakeup << USB_DEVICE_REMOTE_WAKEUP;
+ break;
+ case USB_RECIP_INTERFACE:
+ return cdns2_ep0_delegate_req(pdev);
+ case USB_RECIP_ENDPOINT:
+ ep_sts = cdns2_w_index_to_ep_index(le16_to_cpu(ctrl->wIndex));
+ pep = &pdev->eps[ep_sts];
+
+ if (pep->ep_state & EP_STALLED)
+ status = BIT(USB_ENDPOINT_HALT);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ put_unaligned_le16(status, (__le16 *)pdev->ep0_preq.request.buf);
+
+ cdns2_ep0_enqueue(pdev, pdev->ep0_preq.request.dma,
+ sizeof(*response_pkt), 0);
+
+ return 0;
+}
+
+static int cdns2_ep0_handle_feature_device(struct cdns2_device *pdev,
+ struct usb_ctrlrequest *ctrl,
+ int set)
+{
+ enum usb_device_state state;
+ enum usb_device_speed speed;
+ int ret = 0;
+ u32 wValue;
+ u16 tmode;
+
+ wValue = le16_to_cpu(ctrl->wValue);
+ state = pdev->gadget.state;
+ speed = pdev->gadget.speed;
+
+ switch (wValue) {
+ case USB_DEVICE_REMOTE_WAKEUP:
+ pdev->may_wakeup = !!set;
+ break;
+ case USB_DEVICE_TEST_MODE:
+ if (state != USB_STATE_CONFIGURED || speed > USB_SPEED_HIGH)
+ return -EINVAL;
+
+ tmode = le16_to_cpu(ctrl->wIndex);
+
+ if (!set || (tmode & 0xff) != 0)
+ return -EINVAL;
+
+ tmode >>= 8;
+ switch (tmode) {
+ case USB_TEST_J:
+ case USB_TEST_K:
+ case USB_TEST_SE0_NAK:
+ case USB_TEST_PACKET:
+ /*
+ * The USBHS controller automatically handles the
+ * Set_Feature(testmode) request. Standard test modes
+ * that use values of test mode selector from
+ * 01h to 04h (Test_J, Test_K, Test_SE0_NAK,
+ * Test_Packet) are supported by the
+ * controller(HS - ack, FS - stall).
+ */
+ break;
+ default:
+ ret = -EINVAL;
+ }
+ break;
+ default:
+ ret = -EINVAL;
+ }
+
+ return ret;
+}
+
+static int cdns2_ep0_handle_feature_intf(struct cdns2_device *pdev,
+ struct usb_ctrlrequest *ctrl,
+ int set)
+{
+ int ret = 0;
+ u32 wValue;
+
+ wValue = le16_to_cpu(ctrl->wValue);
+
+ switch (wValue) {
+ case USB_INTRF_FUNC_SUSPEND:
+ break;
+ default:
+ ret = -EINVAL;
+ }
+
+ return ret;
+}
+
+static int cdns2_ep0_handle_feature_endpoint(struct cdns2_device *pdev,
+ struct usb_ctrlrequest *ctrl,
+ int set)
+{
+ struct cdns2_endpoint *pep;
+ u8 wValue;
+
+ wValue = le16_to_cpu(ctrl->wValue);
+ pep = &pdev->eps[cdns2_w_index_to_ep_index(le16_to_cpu(ctrl->wIndex))];
+
+ if (wValue != USB_ENDPOINT_HALT)
+ return -EINVAL;
+
+ if (!(le16_to_cpu(ctrl->wIndex) & ~USB_DIR_IN))
+ return 0;
+
+ switch (wValue) {
+ case USB_ENDPOINT_HALT:
+ if (set || !(pep->ep_state & EP_WEDGE))
+ return cdns2_halt_endpoint(pdev, pep, set);
+ break;
+ default:
+ dev_warn(pdev->dev, "WARN Incorrect wValue %04x\n", wValue);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int cdns2_req_ep0_handle_feature(struct cdns2_device *pdev,
+ struct usb_ctrlrequest *ctrl,
+ int set)
+{
+ switch (ctrl->bRequestType & USB_RECIP_MASK) {
+ case USB_RECIP_DEVICE:
+ return cdns2_ep0_handle_feature_device(pdev, ctrl, set);
+ case USB_RECIP_INTERFACE:
+ return cdns2_ep0_handle_feature_intf(pdev, ctrl, set);
+ case USB_RECIP_ENDPOINT:
+ return cdns2_ep0_handle_feature_endpoint(pdev, ctrl, set);
+ default:
+ return -EINVAL;
+ }
+}
+
+static int cdns2_ep0_std_request(struct cdns2_device *pdev)
+{
+ struct usb_ctrlrequest *ctrl = &pdev->setup;
+ int ret;
+
+ switch (ctrl->bRequest) {
+ case USB_REQ_SET_ADDRESS:
+ ret = cdns2_req_ep0_set_address(pdev,
+ le16_to_cpu(ctrl->wValue));
+ break;
+ case USB_REQ_SET_CONFIGURATION:
+ ret = cdns2_req_ep0_set_configuration(pdev, ctrl);
+ break;
+ case USB_REQ_GET_STATUS:
+ ret = cdns2_req_ep0_handle_status(pdev, ctrl);
+ break;
+ case USB_REQ_CLEAR_FEATURE:
+ ret = cdns2_req_ep0_handle_feature(pdev, ctrl, 0);
+ break;
+ case USB_REQ_SET_FEATURE:
+ ret = cdns2_req_ep0_handle_feature(pdev, ctrl, 1);
+ break;
+ default:
+ ret = cdns2_ep0_delegate_req(pdev);
+ break;
+ }
+
+ return ret;
+}
+
+static void __pending_setup_status_handler(struct cdns2_device *pdev)
+{
+ struct usb_request *request = pdev->pending_status_request;
+
+ if (pdev->status_completion_no_call && request && request->complete) {
+ request->complete(&pdev->eps[0].endpoint, request);
+ pdev->status_completion_no_call = 0;
+ }
+}
+
+void cdns2_pending_setup_status_handler(struct work_struct *work)
+{
+ struct cdns2_device *pdev = container_of(work, struct cdns2_device,
+ pending_status_wq);
+ unsigned long flags;
+
+ spin_lock_irqsave(&pdev->lock, flags);
+ __pending_setup_status_handler(pdev);
+ spin_unlock_irqrestore(&pdev->lock, flags);
+}
+
+void cdns2_handle_setup_packet(struct cdns2_device *pdev)
+{
+ struct usb_ctrlrequest *ctrl = &pdev->setup;
+ struct cdns2_endpoint *pep = &pdev->eps[0];
+ struct cdns2_request *preq;
+ int ret = 0;
+ u16 len;
+ u8 reg;
+ int i;
+
+ writeb(EP0CS_CHGSET, &pdev->ep0_regs->cs);
+
+ for (i = 0; i < 8; i++)
+ ((u8 *)&pdev->setup)[i] = readb(&pdev->ep0_regs->setupdat[i]);
+
+ /*
+ * If SETUP packet was modified while reading just simple ignore it.
+ * The new one will be handled latter.
+ */
+ if (cdns2_check_new_setup(pdev)) {
+ trace_cdns2_ep0_setup("overridden");
+ return;
+ }
+
+ trace_cdns2_ctrl_req(ctrl);
+
+ if (!pdev->gadget_driver)
+ goto out;
+
+ if (pdev->gadget.state == USB_STATE_NOTATTACHED) {
+ dev_err(pdev->dev, "ERR: Setup detected in unattached state\n");
+ ret = -EINVAL;
+ goto out;
+ }
+
+ pep = &pdev->eps[0];
+
+ /* Halt for Ep0 is cleared automatically when SETUP packet arrives. */
+ pep->ep_state &= ~EP_STALLED;
+
+ if (!list_empty(&pep->pending_list)) {
+ preq = cdns2_next_preq(&pep->pending_list);
+ cdns2_gadget_giveback(pep, preq, -ECONNRESET);
+ }
+
+ len = le16_to_cpu(ctrl->wLength);
+ if (len)
+ pdev->ep0_stage = CDNS2_DATA_STAGE;
+ else
+ pdev->ep0_stage = CDNS2_STATUS_STAGE;
+
+ pep->dir = ctrl->bRequestType & USB_DIR_IN;
+
+ /*
+ * SET_ADDRESS request is acknowledged automatically by controller and
+ * in the worse case driver may not notice this request. To check
+ * whether this request has been processed driver can use
+ * fnaddr register.
+ */
+ reg = readb(&pdev->usb_regs->fnaddr);
+ if (pdev->setup.bRequest != USB_REQ_SET_ADDRESS &&
+ pdev->dev_address != reg)
+ cdns2_req_ep0_set_address(pdev, reg);
+
+ if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD)
+ ret = cdns2_ep0_std_request(pdev);
+ else
+ ret = cdns2_ep0_delegate_req(pdev);
+
+ if (ret == USB_GADGET_DELAYED_STATUS) {
+ trace_cdns2_ep0_status_stage("delayed");
+ return;
+ }
+
+out:
+ if (ret < 0)
+ cdns2_ep0_stall(pdev);
+ else if (pdev->ep0_stage == CDNS2_STATUS_STAGE)
+ cdns2_status_stage(pdev);
+}
+
+static void cdns2_transfer_completed(struct cdns2_device *pdev)
+{
+ struct cdns2_endpoint *pep = &pdev->eps[0];
+
+ if (!list_empty(&pep->pending_list)) {
+ struct cdns2_request *preq;
+
+ trace_cdns2_complete_trb(pep, pep->ring.trbs);
+ preq = cdns2_next_preq(&pep->pending_list);
+
+ preq->request.actual =
+ TRB_LEN(le32_to_cpu(pep->ring.trbs->length));
+ cdns2_gadget_giveback(pep, preq, 0);
+ }
+
+ cdns2_status_stage(pdev);
+}
+
+void cdns2_handle_ep0_interrupt(struct cdns2_device *pdev, int dir)
+{
+ u32 ep_sts_reg;
+
+ cdns2_select_ep(pdev, dir);
+
+ trace_cdns2_ep0_irq(pdev);
+
+ ep_sts_reg = readl(&pdev->adma_regs->ep_sts);
+ writel(ep_sts_reg, &pdev->adma_regs->ep_sts);
+
+ __pending_setup_status_handler(pdev);
+
+ if ((ep_sts_reg & DMA_EP_STS_IOC) || (ep_sts_reg & DMA_EP_STS_ISP)) {
+ pdev->eps[0].dir = dir;
+ cdns2_transfer_completed(pdev);
+ }
+}
+
+/*
+ * Function shouldn't be called by gadget driver,
+ * endpoint 0 is allways active.
+ */
+static int cdns2_gadget_ep0_enable(struct usb_ep *ep,
+ const struct usb_endpoint_descriptor *desc)
+{
+ return -EINVAL;
+}
+
+/*
+ * Function shouldn't be called by gadget driver,
+ * endpoint 0 is allways active.
+ */
+static int cdns2_gadget_ep0_disable(struct usb_ep *ep)
+{
+ return -EINVAL;
+}
+
+static int cdns2_gadget_ep0_set_halt(struct usb_ep *ep, int value)
+{
+ struct cdns2_endpoint *pep = ep_to_cdns2_ep(ep);
+ struct cdns2_device *pdev = pep->pdev;
+ unsigned long flags;
+
+ if (!value)
+ return 0;
+
+ spin_lock_irqsave(&pdev->lock, flags);
+ cdns2_ep0_stall(pdev);
+ spin_unlock_irqrestore(&pdev->lock, flags);
+
+ return 0;
+}
+
+static int cdns2_gadget_ep0_set_wedge(struct usb_ep *ep)
+{
+ return cdns2_gadget_ep0_set_halt(ep, 1);
+}
+
+static int cdns2_gadget_ep0_queue(struct usb_ep *ep,
+ struct usb_request *request,
+ gfp_t gfp_flags)
+{
+ struct cdns2_endpoint *pep = ep_to_cdns2_ep(ep);
+ struct cdns2_device *pdev = pep->pdev;
+ struct cdns2_request *preq;
+ unsigned long flags;
+ u8 zlp = 0;
+ int ret;
+
+ spin_lock_irqsave(&pdev->lock, flags);
+
+ preq = to_cdns2_request(request);
+
+ trace_cdns2_request_enqueue(preq);
+
+ /* Cancel the request if controller receive new SETUP packet. */
+ if (cdns2_check_new_setup(pdev)) {
+ trace_cdns2_ep0_setup("overridden");
+ spin_unlock_irqrestore(&pdev->lock, flags);
+ return -ECONNRESET;
+ }
+
+ /* Send STATUS stage. Should be called only for SET_CONFIGURATION. */
+ if (pdev->ep0_stage == CDNS2_STATUS_STAGE) {
+ cdns2_status_stage(pdev);
+
+ request->actual = 0;
+ pdev->status_completion_no_call = true;
+ pdev->pending_status_request = request;
+ usb_gadget_set_state(&pdev->gadget, USB_STATE_CONFIGURED);
+ spin_unlock_irqrestore(&pdev->lock, flags);
+
+ /*
+ * Since there is no completion interrupt for status stage,
+ * it needs to call ->completion in software after
+ * cdns2_gadget_ep0_queue is back.
+ */
+ queue_work(system_freezable_wq, &pdev->pending_status_wq);
+ return 0;
+ }
+
+ if (!list_empty(&pep->pending_list)) {
+ trace_cdns2_ep0_setup("pending");
+ dev_err(pdev->dev,
+ "can't handle multiple requests for ep0\n");
+ spin_unlock_irqrestore(&pdev->lock, flags);
+ return -EBUSY;
+ }
+
+ ret = usb_gadget_map_request_by_dev(pdev->dev, request, pep->dir);
+ if (ret) {
+ spin_unlock_irqrestore(&pdev->lock, flags);
+ dev_err(pdev->dev, "failed to map request\n");
+ return -EINVAL;
+ }
+
+ request->status = -EINPROGRESS;
+ list_add_tail(&preq->list, &pep->pending_list);
+
+ if (request->zero && request->length &&
+ (request->length % ep->maxpacket == 0))
+ zlp = 1;
+
+ cdns2_ep0_enqueue(pdev, request->dma, request->length, zlp);
+
+ spin_unlock_irqrestore(&pdev->lock, flags);
+
+ return 0;
+}
+
+static const struct usb_ep_ops cdns2_gadget_ep0_ops = {
+ .enable = cdns2_gadget_ep0_enable,
+ .disable = cdns2_gadget_ep0_disable,
+ .alloc_request = cdns2_gadget_ep_alloc_request,
+ .free_request = cdns2_gadget_ep_free_request,
+ .queue = cdns2_gadget_ep0_queue,
+ .dequeue = cdns2_gadget_ep_dequeue,
+ .set_halt = cdns2_gadget_ep0_set_halt,
+ .set_wedge = cdns2_gadget_ep0_set_wedge,
+};
+
+void cdns2_ep0_config(struct cdns2_device *pdev)
+{
+ struct cdns2_endpoint *pep;
+
+ pep = &pdev->eps[0];
+
+ if (!list_empty(&pep->pending_list)) {
+ struct cdns2_request *preq;
+
+ preq = cdns2_next_preq(&pep->pending_list);
+ list_del_init(&preq->list);
+ }
+
+ writeb(EP0_FIFO_AUTO, &pdev->ep0_regs->fifo);
+ cdns2_select_ep(pdev, USB_DIR_OUT);
+ writel(DMA_EP_CFG_ENABLE, &pdev->adma_regs->ep_cfg);
+
+ writeb(EP0_FIFO_IO_TX | EP0_FIFO_AUTO, &pdev->ep0_regs->fifo);
+ cdns2_select_ep(pdev, USB_DIR_IN);
+ writel(DMA_EP_CFG_ENABLE, &pdev->adma_regs->ep_cfg);
+
+ writeb(pdev->gadget.ep0->maxpacket, &pdev->ep0_regs->maxpack);
+ writel(DMA_EP_IEN_EP_OUT0 | DMA_EP_IEN_EP_IN0,
+ &pdev->adma_regs->ep_ien);
+}
+
+void cdns2_init_ep0(struct cdns2_device *pdev,
+ struct cdns2_endpoint *pep)
+{
+ u16 maxpacket = le16_to_cpu(cdns2_gadget_ep0_desc.wMaxPacketSize);
+
+ usb_ep_set_maxpacket_limit(&pep->endpoint, maxpacket);
+
+ pep->endpoint.ops = &cdns2_gadget_ep0_ops;
+ pep->endpoint.desc = &cdns2_gadget_ep0_desc;
+ pep->endpoint.caps.type_control = true;
+ pep->endpoint.caps.dir_in = true;
+ pep->endpoint.caps.dir_out = true;
+
+ pdev->gadget.ep0 = &pep->endpoint;
+}
diff --git a/drivers/usb/gadget/udc/cdns2/cdns2-gadget.c b/drivers/usb/gadget/udc/cdns2/cdns2-gadget.c
new file mode 100644
index 0000000000..0eed0e0384
--- /dev/null
+++ b/drivers/usb/gadget/udc/cdns2/cdns2-gadget.c
@@ -0,0 +1,2474 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Cadence USBHS-DEV Driver - gadget side.
+ *
+ * Copyright (C) 2023 Cadence Design Systems.
+ *
+ * Authors: Pawel Laszczak <pawell@cadence.com>
+ */
+
+/*
+ * Work around 1:
+ * At some situations, the controller may get stale data address in TRB
+ * at below sequences:
+ * 1. Controller read TRB includes data address
+ * 2. Software updates TRBs includes data address and Cycle bit
+ * 3. Controller read TRB which includes Cycle bit
+ * 4. DMA run with stale data address
+ *
+ * To fix this problem, driver needs to make the first TRB in TD as invalid.
+ * After preparing all TRBs driver needs to check the position of DMA and
+ * if the DMA point to the first just added TRB and doorbell is 1,
+ * then driver must defer making this TRB as valid. This TRB will be make
+ * as valid during adding next TRB only if DMA is stopped or at TRBERR
+ * interrupt.
+ *
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/pm_runtime.h>
+#include <linux/interrupt.h>
+#include <linux/property.h>
+#include <linux/dmapool.h>
+#include <linux/iopoll.h>
+
+#include "cdns2-gadget.h"
+#include "cdns2-trace.h"
+
+/**
+ * set_reg_bit_32 - set bit in given 32 bits register.
+ * @ptr: register address.
+ * @mask: bits to set.
+ */
+static void set_reg_bit_32(void __iomem *ptr, u32 mask)
+{
+ mask = readl(ptr) | mask;
+ writel(mask, ptr);
+}
+
+/*
+ * clear_reg_bit_32 - clear bit in given 32 bits register.
+ * @ptr: register address.
+ * @mask: bits to clear.
+ */
+static void clear_reg_bit_32(void __iomem *ptr, u32 mask)
+{
+ mask = readl(ptr) & ~mask;
+ writel(mask, ptr);
+}
+
+/* Clear bit in given 8 bits register. */
+static void clear_reg_bit_8(void __iomem *ptr, u8 mask)
+{
+ mask = readb(ptr) & ~mask;
+ writeb(mask, ptr);
+}
+
+/* Set bit in given 16 bits register. */
+void set_reg_bit_8(void __iomem *ptr, u8 mask)
+{
+ mask = readb(ptr) | mask;
+ writeb(mask, ptr);
+}
+
+static int cdns2_get_dma_pos(struct cdns2_device *pdev,
+ struct cdns2_endpoint *pep)
+{
+ int dma_index;
+
+ dma_index = readl(&pdev->adma_regs->ep_traddr) - pep->ring.dma;
+
+ return dma_index / TRB_SIZE;
+}
+
+/* Get next private request from list. */
+struct cdns2_request *cdns2_next_preq(struct list_head *list)
+{
+ return list_first_entry_or_null(list, struct cdns2_request, list);
+}
+
+void cdns2_select_ep(struct cdns2_device *pdev, u32 ep)
+{
+ if (pdev->selected_ep == ep)
+ return;
+
+ pdev->selected_ep = ep;
+ writel(ep, &pdev->adma_regs->ep_sel);
+}
+
+dma_addr_t cdns2_trb_virt_to_dma(struct cdns2_endpoint *pep,
+ struct cdns2_trb *trb)
+{
+ u32 offset = (char *)trb - (char *)pep->ring.trbs;
+
+ return pep->ring.dma + offset;
+}
+
+static void cdns2_free_tr_segment(struct cdns2_endpoint *pep)
+{
+ struct cdns2_device *pdev = pep->pdev;
+ struct cdns2_ring *ring = &pep->ring;
+
+ if (pep->ring.trbs) {
+ dma_pool_free(pdev->eps_dma_pool, ring->trbs, ring->dma);
+ memset(ring, 0, sizeof(*ring));
+ }
+}
+
+/* Allocates Transfer Ring segment. */
+static int cdns2_alloc_tr_segment(struct cdns2_endpoint *pep)
+{
+ struct cdns2_device *pdev = pep->pdev;
+ struct cdns2_trb *link_trb;
+ struct cdns2_ring *ring;
+
+ ring = &pep->ring;
+
+ if (!ring->trbs) {
+ ring->trbs = dma_pool_alloc(pdev->eps_dma_pool,
+ GFP_DMA32 | GFP_ATOMIC,
+ &ring->dma);
+ if (!ring->trbs)
+ return -ENOMEM;
+ }
+
+ memset(ring->trbs, 0, TR_SEG_SIZE);
+
+ if (!pep->num)
+ return 0;
+
+ /* Initialize the last TRB as Link TRB */
+ link_trb = (ring->trbs + (TRBS_PER_SEGMENT - 1));
+ link_trb->buffer = cpu_to_le32(TRB_BUFFER(ring->dma));
+ link_trb->control = cpu_to_le32(TRB_CYCLE | TRB_TYPE(TRB_LINK) |
+ TRB_TOGGLE);
+
+ return 0;
+}
+
+/*
+ * Stalls and flushes selected endpoint.
+ * Endpoint must be selected before invoking this function.
+ */
+static void cdns2_ep_stall_flush(struct cdns2_endpoint *pep)
+{
+ struct cdns2_device *pdev = pep->pdev;
+ int val;
+
+ trace_cdns2_ep_halt(pep, 1, 1);
+
+ writel(DMA_EP_CMD_DFLUSH, &pdev->adma_regs->ep_cmd);
+
+ /* Wait for DFLUSH cleared. */
+ readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val,
+ !(val & DMA_EP_CMD_DFLUSH), 1, 1000);
+ pep->ep_state |= EP_STALLED;
+ pep->ep_state &= ~EP_STALL_PENDING;
+}
+
+/*
+ * Increment a trb index.
+ *
+ * The index should never point to the last link TRB in TR. After incrementing,
+ * if it point to the link TRB, wrap around to the beginning and revert
+ * cycle state bit. The link TRB is always at the last TRB entry.
+ */
+static void cdns2_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
+{
+ (*index)++;
+ if (*index == (trb_in_seg - 1)) {
+ *index = 0;
+ *cs ^= 1;
+ }
+}
+
+static void cdns2_ep_inc_enq(struct cdns2_ring *ring)
+{
+ ring->free_trbs--;
+ cdns2_ep_inc_trb(&ring->enqueue, &ring->pcs, TRBS_PER_SEGMENT);
+}
+
+static void cdns2_ep_inc_deq(struct cdns2_ring *ring)
+{
+ ring->free_trbs++;
+ cdns2_ep_inc_trb(&ring->dequeue, &ring->ccs, TRBS_PER_SEGMENT);
+}
+
+/*
+ * Enable/disable LPM.
+ *
+ * If bit USBCS_LPMNYET is not set and device receive Extended Token packet,
+ * then controller answer with ACK handshake.
+ * If bit USBCS_LPMNYET is set and device receive Extended Token packet,
+ * then controller answer with NYET handshake.
+ */
+static void cdns2_enable_l1(struct cdns2_device *pdev, int enable)
+{
+ if (enable) {
+ clear_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_LPMNYET);
+ writeb(LPMCLOCK_SLEEP_ENTRY, &pdev->usb_regs->lpmclock);
+ } else {
+ set_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_LPMNYET);
+ }
+}
+
+static enum usb_device_speed cdns2_get_speed(struct cdns2_device *pdev)
+{
+ u8 speed = readb(&pdev->usb_regs->speedctrl);
+
+ if (speed & SPEEDCTRL_HS)
+ return USB_SPEED_HIGH;
+ else if (speed & SPEEDCTRL_FS)
+ return USB_SPEED_FULL;
+
+ return USB_SPEED_UNKNOWN;
+}
+
+static struct cdns2_trb *cdns2_next_trb(struct cdns2_endpoint *pep,
+ struct cdns2_trb *trb)
+{
+ if (trb == (pep->ring.trbs + (TRBS_PER_SEGMENT - 1)))
+ return pep->ring.trbs;
+ else
+ return ++trb;
+}
+
+void cdns2_gadget_giveback(struct cdns2_endpoint *pep,
+ struct cdns2_request *preq,
+ int status)
+{
+ struct usb_request *request = &preq->request;
+ struct cdns2_device *pdev = pep->pdev;
+
+ list_del_init(&preq->list);
+
+ if (request->status == -EINPROGRESS)
+ request->status = status;
+
+ usb_gadget_unmap_request_by_dev(pdev->dev, request, pep->dir);
+
+ /* All TRBs have finished, clear the counter. */
+ preq->finished_trb = 0;
+
+ trace_cdns2_request_giveback(preq);
+
+ if (request->complete) {
+ spin_unlock(&pdev->lock);
+ usb_gadget_giveback_request(&pep->endpoint, request);
+ spin_lock(&pdev->lock);
+ }
+
+ if (request->buf == pdev->zlp_buf)
+ cdns2_gadget_ep_free_request(&pep->endpoint, request);
+}
+
+static void cdns2_wa1_restore_cycle_bit(struct cdns2_endpoint *pep)
+{
+ /* Work around for stale data address in TRB. */
+ if (pep->wa1_set) {
+ trace_cdns2_wa1(pep, "restore cycle bit");
+
+ pep->wa1_set = 0;
+ pep->wa1_trb_index = 0xFFFF;
+ if (pep->wa1_cycle_bit)
+ pep->wa1_trb->control |= cpu_to_le32(0x1);
+ else
+ pep->wa1_trb->control &= cpu_to_le32(~0x1);
+ }
+}
+
+static int cdns2_wa1_update_guard(struct cdns2_endpoint *pep,
+ struct cdns2_trb *trb)
+{
+ struct cdns2_device *pdev = pep->pdev;
+
+ if (!pep->wa1_set) {
+ u32 doorbell;
+
+ doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY);
+
+ if (doorbell) {
+ pep->wa1_cycle_bit = pep->ring.pcs ? TRB_CYCLE : 0;
+ pep->wa1_set = 1;
+ pep->wa1_trb = trb;
+ pep->wa1_trb_index = pep->ring.enqueue;
+ trace_cdns2_wa1(pep, "set guard");
+ return 0;
+ }
+ }
+ return 1;
+}
+
+static void cdns2_wa1_tray_restore_cycle_bit(struct cdns2_device *pdev,
+ struct cdns2_endpoint *pep)
+{
+ int dma_index;
+ u32 doorbell;
+
+ doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY);
+ dma_index = cdns2_get_dma_pos(pdev, pep);
+
+ if (!doorbell || dma_index != pep->wa1_trb_index)
+ cdns2_wa1_restore_cycle_bit(pep);
+}
+
+static int cdns2_prepare_ring(struct cdns2_device *pdev,
+ struct cdns2_endpoint *pep,
+ int num_trbs)
+{
+ struct cdns2_trb *link_trb = NULL;
+ int doorbell, dma_index;
+ struct cdns2_ring *ring;
+ u32 ch_bit = 0;
+
+ ring = &pep->ring;
+
+ if (num_trbs > ring->free_trbs) {
+ pep->ep_state |= EP_RING_FULL;
+ trace_cdns2_no_room_on_ring("Ring full\n");
+ return -ENOBUFS;
+ }
+
+ if ((ring->enqueue + num_trbs) >= (TRBS_PER_SEGMENT - 1)) {
+ doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY);
+ dma_index = cdns2_get_dma_pos(pdev, pep);
+
+ /* Driver can't update LINK TRB if it is current processed. */
+ if (doorbell && dma_index == TRBS_PER_SEGMENT - 1) {
+ pep->ep_state |= EP_DEFERRED_DRDY;
+ return -ENOBUFS;
+ }
+
+ /* Update C bt in Link TRB before starting DMA. */
+ link_trb = ring->trbs + (TRBS_PER_SEGMENT - 1);
+
+ /*
+ * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
+ * that DMA stuck at the LINK TRB.
+ * On the other hand, removing TRB_CHAIN for longer TRs for
+ * epXout cause that DMA stuck after handling LINK TRB.
+ * To eliminate this strange behavioral driver set TRB_CHAIN
+ * bit only for TR size > 2.
+ */
+ if (pep->type == USB_ENDPOINT_XFER_ISOC || TRBS_PER_SEGMENT > 2)
+ ch_bit = TRB_CHAIN;
+
+ link_trb->control = cpu_to_le32(((ring->pcs) ? TRB_CYCLE : 0) |
+ TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit);
+ }
+
+ return 0;
+}
+
+static void cdns2_dbg_request_trbs(struct cdns2_endpoint *pep,
+ struct cdns2_request *preq)
+{
+ struct cdns2_trb *link_trb = pep->ring.trbs + (TRBS_PER_SEGMENT - 1);
+ struct cdns2_trb *trb = preq->trb;
+ int num_trbs = preq->num_of_trb;
+ int i = 0;
+
+ while (i < num_trbs) {
+ trace_cdns2_queue_trb(pep, trb + i);
+ if (trb + i == link_trb) {
+ trb = pep->ring.trbs;
+ num_trbs = num_trbs - i;
+ i = 0;
+ } else {
+ i++;
+ }
+ }
+}
+
+static unsigned int cdns2_count_trbs(struct cdns2_endpoint *pep,
+ u64 addr, u64 len)
+{
+ unsigned int num_trbs = 1;
+
+ if (pep->type == USB_ENDPOINT_XFER_ISOC) {
+ /*
+ * To speed up DMA performance address should not exceed 4KB.
+ * for high bandwidth transfer and driver will split
+ * such buffer into two TRBs.
+ */
+ num_trbs = DIV_ROUND_UP(len +
+ (addr & (TRB_MAX_ISO_BUFF_SIZE - 1)),
+ TRB_MAX_ISO_BUFF_SIZE);
+
+ if (pep->interval > 1)
+ num_trbs = pep->dir ? num_trbs * pep->interval : 1;
+ } else if (pep->dir) {
+ /*
+ * One extra link trb for IN direction.
+ * Sometimes DMA doesn't want advance to next TD and transfer
+ * hangs. This extra Link TRB force DMA to advance to next TD.
+ */
+ num_trbs++;
+ }
+
+ return num_trbs;
+}
+
+static unsigned int cdns2_count_sg_trbs(struct cdns2_endpoint *pep,
+ struct usb_request *req)
+{
+ unsigned int i, len, full_len, num_trbs = 0;
+ struct scatterlist *sg;
+ int trb_len = 0;
+
+ full_len = req->length;
+
+ for_each_sg(req->sg, sg, req->num_sgs, i) {
+ len = sg_dma_len(sg);
+ num_trbs += cdns2_count_trbs(pep, sg_dma_address(sg), len);
+ len = min(len, full_len);
+
+ /*
+ * For HS ISO transfer TRBs should not exceed max packet size.
+ * When DMA is working, and data exceed max packet size then
+ * some data will be read in single mode instead burst mode.
+ * This behavior will drastically reduce the copying speed.
+ * To avoid this we need one or two extra TRBs.
+ * This issue occurs for UVC class with sg_supported = 1
+ * because buffers addresses are not aligned to 1024.
+ */
+ if (pep->type == USB_ENDPOINT_XFER_ISOC) {
+ u8 temp;
+
+ trb_len += len;
+ temp = trb_len >> 10;
+
+ if (temp) {
+ if (trb_len % 1024)
+ num_trbs = num_trbs + temp;
+ else
+ num_trbs = num_trbs + temp - 1;
+
+ trb_len = trb_len - (temp << 10);
+ }
+ }
+
+ full_len -= len;
+ if (full_len == 0)
+ break;
+ }
+
+ return num_trbs;
+}
+
+/*
+ * Function prepares the array with optimized AXI burst value for different
+ * transfer lengths. Controller handles the final data which are less
+ * then AXI burst size as single byte transactions.
+ * e.g.:
+ * Let's assume that driver prepares trb with trb->length 700 and burst size
+ * will be set to 128. In this case the controller will handle a first 512 as
+ * single AXI transaction but the next 188 bytes will be handled
+ * as 47 separate AXI transaction.
+ * The better solution is to use the burst size equal 16 and then we will
+ * have only 25 AXI transaction (10 * 64 + 15 *4).
+ */
+static void cdsn2_isoc_burst_opt(struct cdns2_device *pdev)
+{
+ int axi_burst_option[] = {1, 2, 4, 8, 16, 32, 64, 128};
+ int best_burst;
+ int array_size;
+ int opt_burst;
+ int trb_size;
+ int i, j;
+
+ array_size = ARRAY_SIZE(axi_burst_option);
+
+ for (i = 0; i <= MAX_ISO_SIZE; i++) {
+ trb_size = i / 4;
+ best_burst = trb_size ? trb_size : 1;
+
+ for (j = 0; j < array_size; j++) {
+ opt_burst = trb_size / axi_burst_option[j];
+ opt_burst += trb_size % axi_burst_option[j];
+
+ if (opt_burst < best_burst) {
+ best_burst = opt_burst;
+ pdev->burst_opt[i] = axi_burst_option[j];
+ }
+ }
+ }
+}
+
+static void cdns2_ep_tx_isoc(struct cdns2_endpoint *pep,
+ struct cdns2_request *preq,
+ int num_trbs)
+{
+ struct scatterlist *sg = NULL;
+ u32 remaining_packet_size = 0;
+ struct cdns2_trb *trb;
+ bool first_trb = true;
+ dma_addr_t trb_dma;
+ u32 trb_buff_len;
+ u32 block_length;
+ int td_idx = 0;
+ int split_size;
+ u32 full_len;
+ int enqd_len;
+ int sent_len;
+ int sg_iter;
+ u32 control;
+ int num_tds;
+ u32 length;
+
+ /*
+ * For OUT direction 1 TD per interval is enough
+ * because TRBs are not dumped by controller.
+ */
+ num_tds = pep->dir ? pep->interval : 1;
+ split_size = preq->request.num_sgs ? 1024 : 3072;
+
+ for (td_idx = 0; td_idx < num_tds; td_idx++) {
+ if (preq->request.num_sgs) {
+ sg = preq->request.sg;
+ trb_dma = sg_dma_address(sg);
+ block_length = sg_dma_len(sg);
+ } else {
+ trb_dma = preq->request.dma;
+ block_length = preq->request.length;
+ }
+
+ full_len = preq->request.length;
+ sg_iter = preq->request.num_sgs ? preq->request.num_sgs : 1;
+ remaining_packet_size = split_size;
+
+ for (enqd_len = 0; enqd_len < full_len;
+ enqd_len += trb_buff_len) {
+ if (remaining_packet_size == 0)
+ remaining_packet_size = split_size;
+
+ /*
+ * Calculate TRB length.- buffer can't across 4KB
+ * and max packet size.
+ */
+ trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(trb_dma);
+ trb_buff_len = min(trb_buff_len, remaining_packet_size);
+ trb_buff_len = min(trb_buff_len, block_length);
+
+ if (trb_buff_len > full_len - enqd_len)
+ trb_buff_len = full_len - enqd_len;
+
+ control = TRB_TYPE(TRB_NORMAL);
+
+ /*
+ * For IN direction driver has to set the IOC for
+ * last TRB in last TD.
+ * For OUT direction driver must set IOC and ISP
+ * only for last TRB in each TDs.
+ */
+ if (enqd_len + trb_buff_len >= full_len || !pep->dir)
+ control |= TRB_IOC | TRB_ISP;
+
+ /*
+ * Don't give the first TRB to the hardware (by toggling
+ * the cycle bit) until we've finished creating all the
+ * other TRBs.
+ */
+ if (first_trb) {
+ first_trb = false;
+ if (pep->ring.pcs == 0)
+ control |= TRB_CYCLE;
+ } else {
+ control |= pep->ring.pcs;
+ }
+
+ if (enqd_len + trb_buff_len < full_len)
+ control |= TRB_CHAIN;
+
+ length = TRB_LEN(trb_buff_len) |
+ TRB_BURST(pep->pdev->burst_opt[trb_buff_len]);
+
+ trb = pep->ring.trbs + pep->ring.enqueue;
+ trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
+ trb->length = cpu_to_le32(length);
+ trb->control = cpu_to_le32(control);
+
+ trb_dma += trb_buff_len;
+ sent_len = trb_buff_len;
+
+ if (sg && sent_len >= block_length) {
+ /* New sg entry */
+ --sg_iter;
+ sent_len -= block_length;
+ if (sg_iter != 0) {
+ sg = sg_next(sg);
+ trb_dma = sg_dma_address(sg);
+ block_length = sg_dma_len(sg);
+ }
+ }
+
+ remaining_packet_size -= trb_buff_len;
+ block_length -= sent_len;
+ preq->end_trb = pep->ring.enqueue;
+
+ cdns2_ep_inc_enq(&pep->ring);
+ }
+ }
+}
+
+static void cdns2_ep_tx_bulk(struct cdns2_endpoint *pep,
+ struct cdns2_request *preq,
+ int trbs_per_td)
+{
+ struct scatterlist *sg = NULL;
+ struct cdns2_ring *ring;
+ struct cdns2_trb *trb;
+ dma_addr_t trb_dma;
+ int sg_iter = 0;
+ u32 control;
+ u32 length;
+
+ if (preq->request.num_sgs) {
+ sg = preq->request.sg;
+ trb_dma = sg_dma_address(sg);
+ length = sg_dma_len(sg);
+ } else {
+ trb_dma = preq->request.dma;
+ length = preq->request.length;
+ }
+
+ ring = &pep->ring;
+
+ for (sg_iter = 0; sg_iter < trbs_per_td; sg_iter++) {
+ control = TRB_TYPE(TRB_NORMAL) | ring->pcs | TRB_ISP;
+ trb = pep->ring.trbs + ring->enqueue;
+
+ if (pep->dir && sg_iter == trbs_per_td - 1) {
+ preq->end_trb = ring->enqueue;
+ control = ring->pcs | TRB_TYPE(TRB_LINK) | TRB_CHAIN
+ | TRB_IOC;
+ cdns2_ep_inc_enq(&pep->ring);
+
+ if (ring->enqueue == 0)
+ control |= TRB_TOGGLE;
+
+ /* Point to next bad TRB. */
+ trb->buffer = cpu_to_le32(pep->ring.dma +
+ (ring->enqueue * TRB_SIZE));
+ trb->length = 0;
+ trb->control = cpu_to_le32(control);
+ break;
+ }
+
+ /*
+ * Don't give the first TRB to the hardware (by toggling
+ * the cycle bit) until we've finished creating all the
+ * other TRBs.
+ */
+ if (sg_iter == 0)
+ control = control ^ TRB_CYCLE;
+
+ /* For last TRB in TD. */
+ if (sg_iter == (trbs_per_td - (pep->dir ? 2 : 1)))
+ control |= TRB_IOC;
+ else
+ control |= TRB_CHAIN;
+
+ trb->buffer = cpu_to_le32(trb_dma);
+ trb->length = cpu_to_le32(TRB_BURST(pep->trb_burst_size) |
+ TRB_LEN(length));
+ trb->control = cpu_to_le32(control);
+
+ if (sg && sg_iter < (trbs_per_td - 1)) {
+ sg = sg_next(sg);
+ trb_dma = sg_dma_address(sg);
+ length = sg_dma_len(sg);
+ }
+
+ preq->end_trb = ring->enqueue;
+ cdns2_ep_inc_enq(&pep->ring);
+ }
+}
+
+static void cdns2_set_drdy(struct cdns2_device *pdev,
+ struct cdns2_endpoint *pep)
+{
+ trace_cdns2_ring(pep);
+
+ /*
+ * Memory barrier - Cycle Bit must be set before doorbell.
+ */
+ dma_wmb();
+
+ /* Clearing TRBERR and DESCMIS before setting DRDY. */
+ writel(DMA_EP_STS_TRBERR | DMA_EP_STS_DESCMIS,
+ &pdev->adma_regs->ep_sts);
+ writel(DMA_EP_CMD_DRDY, &pdev->adma_regs->ep_cmd);
+
+ if (readl(&pdev->adma_regs->ep_sts) & DMA_EP_STS_TRBERR) {
+ writel(DMA_EP_STS_TRBERR, &pdev->adma_regs->ep_sts);
+ writel(DMA_EP_CMD_DRDY, &pdev->adma_regs->ep_cmd);
+ }
+
+ trace_cdns2_doorbell_epx(pep, readl(&pdev->adma_regs->ep_traddr));
+}
+
+static int cdns2_prepare_first_isoc_transfer(struct cdns2_device *pdev,
+ struct cdns2_endpoint *pep)
+{
+ struct cdns2_trb *trb;
+ u32 buffer;
+ u8 hw_ccs;
+
+ if ((readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY))
+ return -EBUSY;
+
+ if (!pep->dir) {
+ set_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE);
+ writel(pep->ring.dma + pep->ring.dequeue,
+ &pdev->adma_regs->ep_traddr);
+ return 0;
+ }
+
+ /*
+ * The first packet after doorbell can be corrupted so,
+ * driver prepares 0 length packet as first packet.
+ */
+ buffer = pep->ring.dma + pep->ring.dequeue * TRB_SIZE;
+ hw_ccs = !!DMA_EP_STS_CCS(readl(&pdev->adma_regs->ep_sts));
+
+ trb = &pep->ring.trbs[TRBS_PER_SEGMENT];
+ trb->length = 0;
+ trb->buffer = cpu_to_le32(TRB_BUFFER(buffer));
+ trb->control = cpu_to_le32((hw_ccs ? TRB_CYCLE : 0) | TRB_TYPE(TRB_NORMAL));
+
+ /*
+ * LINK TRB is used to force updating cycle bit in controller and
+ * move to correct place in transfer ring.
+ */
+ trb++;
+ trb->length = 0;
+ trb->buffer = cpu_to_le32(TRB_BUFFER(buffer));
+ trb->control = cpu_to_le32((hw_ccs ? TRB_CYCLE : 0) |
+ TRB_TYPE(TRB_LINK) | TRB_CHAIN);
+
+ if (hw_ccs != pep->ring.ccs)
+ trb->control |= cpu_to_le32(TRB_TOGGLE);
+
+ set_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE);
+ writel(pep->ring.dma + (TRBS_PER_SEGMENT * TRB_SIZE),
+ &pdev->adma_regs->ep_traddr);
+
+ return 0;
+}
+
+/* Prepare and start transfer on no-default endpoint. */
+static int cdns2_ep_run_transfer(struct cdns2_endpoint *pep,
+ struct cdns2_request *preq)
+{
+ struct cdns2_device *pdev = pep->pdev;
+ struct cdns2_ring *ring;
+ u32 togle_pcs = 1;
+ int num_trbs;
+ int ret;
+
+ cdns2_select_ep(pdev, pep->endpoint.address);
+
+ if (preq->request.sg)
+ num_trbs = cdns2_count_sg_trbs(pep, &preq->request);
+ else
+ num_trbs = cdns2_count_trbs(pep, preq->request.dma,
+ preq->request.length);
+
+ ret = cdns2_prepare_ring(pdev, pep, num_trbs);
+ if (ret)
+ return ret;
+
+ ring = &pep->ring;
+ preq->start_trb = ring->enqueue;
+ preq->trb = ring->trbs + ring->enqueue;
+
+ if (usb_endpoint_xfer_isoc(pep->endpoint.desc)) {
+ cdns2_ep_tx_isoc(pep, preq, num_trbs);
+ } else {
+ togle_pcs = cdns2_wa1_update_guard(pep, ring->trbs + ring->enqueue);
+ cdns2_ep_tx_bulk(pep, preq, num_trbs);
+ }
+
+ preq->num_of_trb = num_trbs;
+
+ /*
+ * Memory barrier - cycle bit must be set as the last operation.
+ */
+ dma_wmb();
+
+ /* Give the TD to the consumer. */
+ if (togle_pcs)
+ preq->trb->control = preq->trb->control ^ cpu_to_le32(1);
+
+ cdns2_wa1_tray_restore_cycle_bit(pdev, pep);
+ cdns2_dbg_request_trbs(pep, preq);
+
+ if (!pep->wa1_set && !(pep->ep_state & EP_STALLED) && !pep->skip) {
+ if (pep->type == USB_ENDPOINT_XFER_ISOC) {
+ ret = cdns2_prepare_first_isoc_transfer(pdev, pep);
+ if (ret)
+ return 0;
+ }
+
+ cdns2_set_drdy(pdev, pep);
+ }
+
+ return 0;
+}
+
+/* Prepare and start transfer for all not started requests. */
+static int cdns2_start_all_request(struct cdns2_device *pdev,
+ struct cdns2_endpoint *pep)
+{
+ struct cdns2_request *preq;
+ int ret;
+
+ while (!list_empty(&pep->deferred_list)) {
+ preq = cdns2_next_preq(&pep->deferred_list);
+
+ ret = cdns2_ep_run_transfer(pep, preq);
+ if (ret)
+ return ret;
+
+ list_move_tail(&preq->list, &pep->pending_list);
+ }
+
+ pep->ep_state &= ~EP_RING_FULL;
+
+ return 0;
+}
+
+/*
+ * Check whether trb has been handled by DMA.
+ *
+ * Endpoint must be selected before invoking this function.
+ *
+ * Returns false if request has not been handled by DMA, else returns true.
+ *
+ * SR - start ring
+ * ER - end ring
+ * DQ = ring->dequeue - dequeue position
+ * EQ = ring->enqueue - enqueue position
+ * ST = preq->start_trb - index of first TRB in transfer ring
+ * ET = preq->end_trb - index of last TRB in transfer ring
+ * CI = current_index - index of processed TRB by DMA.
+ *
+ * As first step, we check if the TRB between the ST and ET.
+ * Then, we check if cycle bit for index pep->dequeue
+ * is correct.
+ *
+ * some rules:
+ * 1. ring->dequeue never equals to current_index.
+ * 2 ring->enqueue never exceed ring->dequeue
+ * 3. exception: ring->enqueue == ring->dequeue
+ * and ring->free_trbs is zero.
+ * This case indicate that TR is full.
+ *
+ * At below two cases, the request have been handled.
+ * Case 1 - ring->dequeue < current_index
+ * SR ... EQ ... DQ ... CI ... ER
+ * SR ... DQ ... CI ... EQ ... ER
+ *
+ * Case 2 - ring->dequeue > current_index
+ * This situation takes place when CI go through the LINK TRB at the end of
+ * transfer ring.
+ * SR ... CI ... EQ ... DQ ... ER
+ */
+static bool cdns2_trb_handled(struct cdns2_endpoint *pep,
+ struct cdns2_request *preq)
+{
+ struct cdns2_device *pdev = pep->pdev;
+ struct cdns2_ring *ring;
+ struct cdns2_trb *trb;
+ int current_index = 0;
+ int handled = 0;
+ int doorbell;
+
+ ring = &pep->ring;
+ current_index = cdns2_get_dma_pos(pdev, pep);
+ doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY);
+
+ /*
+ * Only ISO transfer can use 2 entries outside the standard
+ * Transfer Ring. First of them is used as zero length packet and the
+ * second as LINK TRB.
+ */
+ if (current_index >= TRBS_PER_SEGMENT)
+ goto finish;
+
+ /* Current trb doesn't belong to this request. */
+ if (preq->start_trb < preq->end_trb) {
+ if (ring->dequeue > preq->end_trb)
+ goto finish;
+
+ if (ring->dequeue < preq->start_trb)
+ goto finish;
+ }
+
+ if (preq->start_trb > preq->end_trb && ring->dequeue > preq->end_trb &&
+ ring->dequeue < preq->start_trb)
+ goto finish;
+
+ if (preq->start_trb == preq->end_trb && ring->dequeue != preq->end_trb)
+ goto finish;
+
+ trb = &ring->trbs[ring->dequeue];
+
+ if ((le32_to_cpu(trb->control) & TRB_CYCLE) != ring->ccs)
+ goto finish;
+
+ if (doorbell == 1 && current_index == ring->dequeue)
+ goto finish;
+
+ /* The corner case for TRBS_PER_SEGMENT equal 2). */
+ if (TRBS_PER_SEGMENT == 2 && pep->type != USB_ENDPOINT_XFER_ISOC) {
+ handled = 1;
+ goto finish;
+ }
+
+ if (ring->enqueue == ring->dequeue &&
+ ring->free_trbs == 0) {
+ handled = 1;
+ } else if (ring->dequeue < current_index) {
+ if ((current_index == (TRBS_PER_SEGMENT - 1)) &&
+ !ring->dequeue)
+ goto finish;
+
+ handled = 1;
+ } else if (ring->dequeue > current_index) {
+ handled = 1;
+ }
+
+finish:
+ trace_cdns2_request_handled(preq, current_index, handled);
+
+ return handled;
+}
+
+static void cdns2_skip_isoc_td(struct cdns2_device *pdev,
+ struct cdns2_endpoint *pep,
+ struct cdns2_request *preq)
+{
+ struct cdns2_trb *trb;
+ int i;
+
+ trb = pep->ring.trbs + pep->ring.dequeue;
+
+ for (i = preq->finished_trb ; i < preq->num_of_trb; i++) {
+ preq->finished_trb++;
+ trace_cdns2_complete_trb(pep, trb);
+ cdns2_ep_inc_deq(&pep->ring);
+ trb = cdns2_next_trb(pep, trb);
+ }
+
+ cdns2_gadget_giveback(pep, preq, 0);
+ cdns2_prepare_first_isoc_transfer(pdev, pep);
+ pep->skip = false;
+ cdns2_set_drdy(pdev, pep);
+}
+
+static void cdns2_transfer_completed(struct cdns2_device *pdev,
+ struct cdns2_endpoint *pep)
+{
+ struct cdns2_request *preq = NULL;
+ bool request_handled = false;
+ struct cdns2_trb *trb;
+
+ while (!list_empty(&pep->pending_list)) {
+ preq = cdns2_next_preq(&pep->pending_list);
+ trb = pep->ring.trbs + pep->ring.dequeue;
+
+ /*
+ * The TRB was changed as link TRB, and the request
+ * was handled at ep_dequeue.
+ */
+ while (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK &&
+ le32_to_cpu(trb->length)) {
+ trace_cdns2_complete_trb(pep, trb);
+ cdns2_ep_inc_deq(&pep->ring);
+ trb = pep->ring.trbs + pep->ring.dequeue;
+ }
+
+ /*
+ * Re-select endpoint. It could be changed by other CPU
+ * during handling usb_gadget_giveback_request.
+ */
+ cdns2_select_ep(pdev, pep->endpoint.address);
+
+ while (cdns2_trb_handled(pep, preq)) {
+ preq->finished_trb++;
+
+ if (preq->finished_trb >= preq->num_of_trb)
+ request_handled = true;
+
+ trb = pep->ring.trbs + pep->ring.dequeue;
+ trace_cdns2_complete_trb(pep, trb);
+
+ if (pep->dir && pep->type == USB_ENDPOINT_XFER_ISOC)
+ /*
+ * For ISOC IN controller doens't update the
+ * trb->length.
+ */
+ preq->request.actual = preq->request.length;
+ else
+ preq->request.actual +=
+ TRB_LEN(le32_to_cpu(trb->length));
+
+ cdns2_ep_inc_deq(&pep->ring);
+ }
+
+ if (request_handled) {
+ cdns2_gadget_giveback(pep, preq, 0);
+ request_handled = false;
+ } else {
+ goto prepare_next_td;
+ }
+
+ if (pep->type != USB_ENDPOINT_XFER_ISOC &&
+ TRBS_PER_SEGMENT == 2)
+ break;
+ }
+
+prepare_next_td:
+ if (pep->skip && preq)
+ cdns2_skip_isoc_td(pdev, pep, preq);
+
+ if (!(pep->ep_state & EP_STALLED) &&
+ !(pep->ep_state & EP_STALL_PENDING))
+ cdns2_start_all_request(pdev, pep);
+}
+
+static void cdns2_wakeup(struct cdns2_device *pdev)
+{
+ if (!pdev->may_wakeup)
+ return;
+
+ /* Start driving resume signaling to indicate remote wakeup. */
+ set_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_SIGRSUME);
+}
+
+static void cdns2_rearm_transfer(struct cdns2_endpoint *pep, u8 rearm)
+{
+ struct cdns2_device *pdev = pep->pdev;
+
+ cdns2_wa1_restore_cycle_bit(pep);
+
+ if (rearm) {
+ trace_cdns2_ring(pep);
+
+ /* Cycle Bit must be updated before arming DMA. */
+ dma_wmb();
+
+ writel(DMA_EP_CMD_DRDY, &pdev->adma_regs->ep_cmd);
+
+ cdns2_wakeup(pdev);
+
+ trace_cdns2_doorbell_epx(pep,
+ readl(&pdev->adma_regs->ep_traddr));
+ }
+}
+
+static void cdns2_handle_epx_interrupt(struct cdns2_endpoint *pep)
+{
+ struct cdns2_device *pdev = pep->pdev;
+ u8 isoerror = 0;
+ u32 ep_sts_reg;
+ u32 val;
+
+ cdns2_select_ep(pdev, pep->endpoint.address);
+
+ trace_cdns2_epx_irq(pdev, pep);
+
+ ep_sts_reg = readl(&pdev->adma_regs->ep_sts);
+ writel(ep_sts_reg, &pdev->adma_regs->ep_sts);
+
+ if (pep->type == USB_ENDPOINT_XFER_ISOC) {
+ u8 mult;
+ u8 cs;
+
+ mult = USB_EP_MAXP_MULT(pep->endpoint.desc->wMaxPacketSize);
+ cs = pep->dir ? readb(&pdev->epx_regs->ep[pep->num - 1].txcs) :
+ readb(&pdev->epx_regs->ep[pep->num - 1].rxcs);
+ if (mult > 0)
+ isoerror = EPX_CS_ERR(cs);
+ }
+
+ /*
+ * Sometimes ISO Error for mult=1 or mult=2 is not propagated on time
+ * from USB module to DMA module. To protect against this driver
+ * checks also the txcs/rxcs registers.
+ */
+ if ((ep_sts_reg & DMA_EP_STS_ISOERR) || isoerror) {
+ clear_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE);
+
+ /* Wait for DBUSY cleared. */
+ readl_poll_timeout_atomic(&pdev->adma_regs->ep_sts, val,
+ !(val & DMA_EP_STS_DBUSY), 1, 125);
+
+ writel(DMA_EP_CMD_DFLUSH, &pep->pdev->adma_regs->ep_cmd);
+
+ /* Wait for DFLUSH cleared. */
+ readl_poll_timeout_atomic(&pep->pdev->adma_regs->ep_cmd, val,
+ !(val & DMA_EP_CMD_DFLUSH), 1, 10);
+
+ pep->skip = true;
+ }
+
+ if (ep_sts_reg & DMA_EP_STS_TRBERR || pep->skip) {
+ if (pep->ep_state & EP_STALL_PENDING &&
+ !(ep_sts_reg & DMA_EP_STS_DESCMIS))
+ cdns2_ep_stall_flush(pep);
+
+ /*
+ * For isochronous transfer driver completes request on
+ * IOC or on TRBERR. IOC appears only when device receive
+ * OUT data packet. If host disable stream or lost some packet
+ * then the only way to finish all queued transfer is to do it
+ * on TRBERR event.
+ */
+ if (pep->type == USB_ENDPOINT_XFER_ISOC && !pep->wa1_set) {
+ if (!pep->dir)
+ clear_reg_bit_32(&pdev->adma_regs->ep_cfg,
+ DMA_EP_CFG_ENABLE);
+
+ cdns2_transfer_completed(pdev, pep);
+ if (pep->ep_state & EP_DEFERRED_DRDY) {
+ pep->ep_state &= ~EP_DEFERRED_DRDY;
+ cdns2_set_drdy(pdev, pep);
+ }
+
+ return;
+ }
+
+ cdns2_transfer_completed(pdev, pep);
+
+ if (!(pep->ep_state & EP_STALLED) &&
+ !(pep->ep_state & EP_STALL_PENDING)) {
+ if (pep->ep_state & EP_DEFERRED_DRDY) {
+ pep->ep_state &= ~EP_DEFERRED_DRDY;
+ cdns2_start_all_request(pdev, pep);
+ } else {
+ cdns2_rearm_transfer(pep, pep->wa1_set);
+ }
+ }
+
+ return;
+ }
+
+ if ((ep_sts_reg & DMA_EP_STS_IOC) || (ep_sts_reg & DMA_EP_STS_ISP))
+ cdns2_transfer_completed(pdev, pep);
+}
+
+static void cdns2_disconnect_gadget(struct cdns2_device *pdev)
+{
+ if (pdev->gadget_driver && pdev->gadget_driver->disconnect)
+ pdev->gadget_driver->disconnect(&pdev->gadget);
+}
+
+static irqreturn_t cdns2_usb_irq_handler(int irq, void *data)
+{
+ struct cdns2_device *pdev = data;
+ unsigned long reg_ep_ists;
+ u8 reg_usb_irq_m;
+ u8 reg_ext_irq_m;
+ u8 reg_usb_irq;
+ u8 reg_ext_irq;
+
+ if (pdev->in_lpm)
+ return IRQ_NONE;
+
+ reg_usb_irq_m = readb(&pdev->interrupt_regs->usbien);
+ reg_ext_irq_m = readb(&pdev->interrupt_regs->extien);
+
+ /* Mask all sources of interrupt. */
+ writeb(0, &pdev->interrupt_regs->usbien);
+ writeb(0, &pdev->interrupt_regs->extien);
+ writel(0, &pdev->adma_regs->ep_ien);
+
+ /* Clear interrupt sources. */
+ writel(0, &pdev->adma_regs->ep_sts);
+ writeb(0, &pdev->interrupt_regs->usbirq);
+ writeb(0, &pdev->interrupt_regs->extirq);
+
+ reg_ep_ists = readl(&pdev->adma_regs->ep_ists);
+ reg_usb_irq = readb(&pdev->interrupt_regs->usbirq);
+ reg_ext_irq = readb(&pdev->interrupt_regs->extirq);
+
+ if (reg_ep_ists || (reg_usb_irq & reg_usb_irq_m) ||
+ (reg_ext_irq & reg_ext_irq_m))
+ return IRQ_WAKE_THREAD;
+
+ writeb(USB_IEN_INIT, &pdev->interrupt_regs->usbien);
+ writeb(EXTIRQ_WAKEUP, &pdev->interrupt_regs->extien);
+ writel(~0, &pdev->adma_regs->ep_ien);
+
+ return IRQ_NONE;
+}
+
+static irqreturn_t cdns2_thread_usb_irq_handler(struct cdns2_device *pdev)
+{
+ u8 usb_irq, ext_irq;
+ int speed;
+ int i;
+
+ ext_irq = readb(&pdev->interrupt_regs->extirq) & EXTIRQ_WAKEUP;
+ writeb(ext_irq, &pdev->interrupt_regs->extirq);
+
+ usb_irq = readb(&pdev->interrupt_regs->usbirq) & USB_IEN_INIT;
+ writeb(usb_irq, &pdev->interrupt_regs->usbirq);
+
+ if (!ext_irq && !usb_irq)
+ return IRQ_NONE;
+
+ trace_cdns2_usb_irq(usb_irq, ext_irq);
+
+ if (ext_irq & EXTIRQ_WAKEUP) {
+ if (pdev->gadget_driver && pdev->gadget_driver->resume) {
+ spin_unlock(&pdev->lock);
+ pdev->gadget_driver->resume(&pdev->gadget);
+ spin_lock(&pdev->lock);
+ }
+ }
+
+ if (usb_irq & USBIRQ_LPM) {
+ u8 reg = readb(&pdev->usb_regs->lpmctrl);
+
+ /* LPM1 enter */
+ if (!(reg & LPMCTRLLH_LPMNYET))
+ writeb(0, &pdev->usb_regs->sleep_clkgate);
+ }
+
+ if (usb_irq & USBIRQ_SUSPEND) {
+ if (pdev->gadget_driver && pdev->gadget_driver->suspend) {
+ spin_unlock(&pdev->lock);
+ pdev->gadget_driver->suspend(&pdev->gadget);
+ spin_lock(&pdev->lock);
+ }
+ }
+
+ if (usb_irq & USBIRQ_URESET) {
+ if (pdev->gadget_driver) {
+ pdev->dev_address = 0;
+
+ spin_unlock(&pdev->lock);
+ usb_gadget_udc_reset(&pdev->gadget,
+ pdev->gadget_driver);
+ spin_lock(&pdev->lock);
+
+ /*
+ * The USBIRQ_URESET is reported at the beginning of
+ * reset signal. 100ms is enough time to finish reset
+ * process. For high-speed reset procedure is completed
+ * when controller detect HS mode.
+ */
+ for (i = 0; i < 100; i++) {
+ mdelay(1);
+ speed = cdns2_get_speed(pdev);
+ if (speed == USB_SPEED_HIGH)
+ break;
+ }
+
+ pdev->gadget.speed = speed;
+ cdns2_enable_l1(pdev, 0);
+ cdns2_ep0_config(pdev);
+ pdev->may_wakeup = 0;
+ }
+ }
+
+ if (usb_irq & USBIRQ_SUDAV) {
+ pdev->ep0_stage = CDNS2_SETUP_STAGE;
+ cdns2_handle_setup_packet(pdev);
+ }
+
+ return IRQ_HANDLED;
+}
+
+/* Deferred USB interrupt handler. */
+static irqreturn_t cdns2_thread_irq_handler(int irq, void *data)
+{
+ struct cdns2_device *pdev = data;
+ unsigned long dma_ep_ists;
+ unsigned long flags;
+ unsigned int bit;
+
+ local_bh_disable();
+ spin_lock_irqsave(&pdev->lock, flags);
+
+ cdns2_thread_usb_irq_handler(pdev);
+
+ dma_ep_ists = readl(&pdev->adma_regs->ep_ists);
+ if (!dma_ep_ists)
+ goto unlock;
+
+ trace_cdns2_dma_ep_ists(dma_ep_ists);
+
+ /* Handle default endpoint OUT. */
+ if (dma_ep_ists & DMA_EP_ISTS_EP_OUT0)
+ cdns2_handle_ep0_interrupt(pdev, USB_DIR_OUT);
+
+ /* Handle default endpoint IN. */
+ if (dma_ep_ists & DMA_EP_ISTS_EP_IN0)
+ cdns2_handle_ep0_interrupt(pdev, USB_DIR_IN);
+
+ dma_ep_ists &= ~(DMA_EP_ISTS_EP_OUT0 | DMA_EP_ISTS_EP_IN0);
+
+ for_each_set_bit(bit, &dma_ep_ists, sizeof(u32) * BITS_PER_BYTE) {
+ u8 ep_idx = bit > 16 ? (bit - 16) * 2 : (bit * 2) - 1;
+
+ /*
+ * Endpoints in pdev->eps[] are held in order:
+ * ep0, ep1out, ep1in, ep2out, ep2in... ep15out, ep15in.
+ * but in dma_ep_ists in order:
+ * ep0 ep1out ep2out ... ep15out ep0in ep1in .. ep15in
+ */
+ cdns2_handle_epx_interrupt(&pdev->eps[ep_idx]);
+ }
+
+unlock:
+ writel(~0, &pdev->adma_regs->ep_ien);
+ writeb(USB_IEN_INIT, &pdev->interrupt_regs->usbien);
+ writeb(EXTIRQ_WAKEUP, &pdev->interrupt_regs->extien);
+
+ spin_unlock_irqrestore(&pdev->lock, flags);
+ local_bh_enable();
+
+ return IRQ_HANDLED;
+}
+
+/* Calculates and assigns onchip memory for endpoints. */
+static void cdns2_eps_onchip_buffer_init(struct cdns2_device *pdev)
+{
+ struct cdns2_endpoint *pep;
+ int min_buf_tx = 0;
+ int min_buf_rx = 0;
+ u16 tx_offset = 0;
+ u16 rx_offset = 0;
+ int free;
+ int i;
+
+ for (i = 0; i < CDNS2_ENDPOINTS_NUM; i++) {
+ pep = &pdev->eps[i];
+
+ if (!(pep->ep_state & EP_CLAIMED))
+ continue;
+
+ if (pep->dir)
+ min_buf_tx += pep->buffering;
+ else
+ min_buf_rx += pep->buffering;
+ }
+
+ for (i = 0; i < CDNS2_ENDPOINTS_NUM; i++) {
+ pep = &pdev->eps[i];
+
+ if (!(pep->ep_state & EP_CLAIMED))
+ continue;
+
+ if (pep->dir) {
+ free = pdev->onchip_tx_buf - min_buf_tx;
+
+ if (free + pep->buffering >= 4)
+ free = 4;
+ else
+ free = free + pep->buffering;
+
+ min_buf_tx = min_buf_tx - pep->buffering + free;
+
+ pep->buffering = free;
+
+ writel(tx_offset,
+ &pdev->epx_regs->txstaddr[pep->num - 1]);
+ pdev->epx_regs->txstaddr[pep->num - 1] = tx_offset;
+
+ dev_dbg(pdev->dev, "%s onchip address %04x, buffering: %d\n",
+ pep->name, tx_offset, pep->buffering);
+
+ tx_offset += pep->buffering * 1024;
+ } else {
+ free = pdev->onchip_rx_buf - min_buf_rx;
+
+ if (free + pep->buffering >= 4)
+ free = 4;
+ else
+ free = free + pep->buffering;
+
+ min_buf_rx = min_buf_rx - pep->buffering + free;
+
+ pep->buffering = free;
+ writel(rx_offset,
+ &pdev->epx_regs->rxstaddr[pep->num - 1]);
+
+ dev_dbg(pdev->dev, "%s onchip address %04x, buffering: %d\n",
+ pep->name, rx_offset, pep->buffering);
+
+ rx_offset += pep->buffering * 1024;
+ }
+ }
+}
+
+/* Configure hardware endpoint. */
+static int cdns2_ep_config(struct cdns2_endpoint *pep, bool enable)
+{
+ bool is_iso_ep = (pep->type == USB_ENDPOINT_XFER_ISOC);
+ struct cdns2_device *pdev = pep->pdev;
+ u32 max_packet_size;
+ u8 dir = 0;
+ u8 ep_cfg;
+ u8 mult;
+ u32 val;
+ int ret;
+
+ switch (pep->type) {
+ case USB_ENDPOINT_XFER_INT:
+ ep_cfg = EPX_CON_TYPE_INT;
+ break;
+ case USB_ENDPOINT_XFER_BULK:
+ ep_cfg = EPX_CON_TYPE_BULK;
+ break;
+ default:
+ mult = USB_EP_MAXP_MULT(pep->endpoint.desc->wMaxPacketSize);
+ ep_cfg = mult << EPX_CON_ISOD_SHIFT;
+ ep_cfg |= EPX_CON_TYPE_ISOC;
+
+ if (pep->dir) {
+ set_reg_bit_8(&pdev->epx_regs->isoautoarm, BIT(pep->num));
+ set_reg_bit_8(&pdev->epx_regs->isoautodump, BIT(pep->num));
+ set_reg_bit_8(&pdev->epx_regs->isodctrl, BIT(pep->num));
+ }
+ }
+
+ switch (pdev->gadget.speed) {
+ case USB_SPEED_FULL:
+ max_packet_size = is_iso_ep ? 1023 : 64;
+ break;
+ case USB_SPEED_HIGH:
+ max_packet_size = is_iso_ep ? 1024 : 512;
+ break;
+ default:
+ /* All other speed are not supported. */
+ return -EINVAL;
+ }
+
+ ep_cfg |= (EPX_CON_VAL | (pep->buffering - 1));
+
+ if (pep->dir) {
+ dir = FIFOCTRL_IO_TX;
+ writew(max_packet_size, &pdev->epx_regs->txmaxpack[pep->num - 1]);
+ writeb(ep_cfg, &pdev->epx_regs->ep[pep->num - 1].txcon);
+ } else {
+ writew(max_packet_size, &pdev->epx_regs->rxmaxpack[pep->num - 1]);
+ writeb(ep_cfg, &pdev->epx_regs->ep[pep->num - 1].rxcon);
+ }
+
+ writeb(pep->num | dir | FIFOCTRL_FIFOAUTO,
+ &pdev->usb_regs->fifoctrl);
+ writeb(pep->num | dir, &pdev->epx_regs->endprst);
+ writeb(pep->num | ENDPRST_FIFORST | ENDPRST_TOGRST | dir,
+ &pdev->epx_regs->endprst);
+
+ if (max_packet_size == 1024)
+ pep->trb_burst_size = 128;
+ else if (max_packet_size >= 512)
+ pep->trb_burst_size = 64;
+ else
+ pep->trb_burst_size = 16;
+
+ cdns2_select_ep(pdev, pep->num | pep->dir);
+ writel(DMA_EP_CMD_EPRST | DMA_EP_CMD_DFLUSH, &pdev->adma_regs->ep_cmd);
+
+ ret = readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val,
+ !(val & (DMA_EP_CMD_DFLUSH |
+ DMA_EP_CMD_EPRST)),
+ 1, 1000);
+
+ if (ret)
+ return ret;
+
+ writel(DMA_EP_STS_TRBERR | DMA_EP_STS_ISOERR, &pdev->adma_regs->ep_sts_en);
+
+ if (enable)
+ writel(DMA_EP_CFG_ENABLE, &pdev->adma_regs->ep_cfg);
+
+ trace_cdns2_epx_hw_cfg(pdev, pep);
+
+ dev_dbg(pdev->dev, "Configure %s: with MPS: %08x, ep con: %02x\n",
+ pep->name, max_packet_size, ep_cfg);
+
+ return 0;
+}
+
+struct usb_request *cdns2_gadget_ep_alloc_request(struct usb_ep *ep,
+ gfp_t gfp_flags)
+{
+ struct cdns2_endpoint *pep = ep_to_cdns2_ep(ep);
+ struct cdns2_request *preq;
+
+ preq = kzalloc(sizeof(*preq), gfp_flags);
+ if (!preq)
+ return NULL;
+
+ preq->pep = pep;
+
+ trace_cdns2_alloc_request(preq);
+
+ return &preq->request;
+}
+
+void cdns2_gadget_ep_free_request(struct usb_ep *ep,
+ struct usb_request *request)
+{
+ struct cdns2_request *preq = to_cdns2_request(request);
+
+ trace_cdns2_free_request(preq);
+ kfree(preq);
+}
+
+static int cdns2_gadget_ep_enable(struct usb_ep *ep,
+ const struct usb_endpoint_descriptor *desc)
+{
+ u32 reg = DMA_EP_STS_EN_TRBERREN;
+ struct cdns2_endpoint *pep;
+ struct cdns2_device *pdev;
+ unsigned long flags;
+ int enable = 1;
+ int ret = 0;
+
+ if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT ||
+ !desc->wMaxPacketSize) {
+ return -EINVAL;
+ }
+
+ pep = ep_to_cdns2_ep(ep);
+ pdev = pep->pdev;
+
+ if (dev_WARN_ONCE(pdev->dev, pep->ep_state & EP_ENABLED,
+ "%s is already enabled\n", pep->name))
+ return 0;
+
+ spin_lock_irqsave(&pdev->lock, flags);
+
+ pep->type = usb_endpoint_type(desc);
+ pep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
+
+ if (pdev->gadget.speed == USB_SPEED_FULL)
+ if (pep->type == USB_ENDPOINT_XFER_INT)
+ pep->interval = desc->bInterval;
+
+ if (pep->interval > ISO_MAX_INTERVAL &&
+ pep->type == USB_ENDPOINT_XFER_ISOC) {
+ dev_err(pdev->dev, "ISO period is limited to %d (current: %d)\n",
+ ISO_MAX_INTERVAL, pep->interval);
+
+ ret = -EINVAL;
+ goto exit;
+ }
+
+ /*
+ * During ISO OUT traffic DMA reads Transfer Ring for the EP which has
+ * never got doorbell.
+ * This issue was detected only on simulation, but to avoid this issue
+ * driver add protection against it. To fix it driver enable ISO OUT
+ * endpoint before setting DRBL. This special treatment of ISO OUT
+ * endpoints are recommended by controller specification.
+ */
+ if (pep->type == USB_ENDPOINT_XFER_ISOC && !pep->dir)
+ enable = 0;
+
+ ret = cdns2_alloc_tr_segment(pep);
+ if (ret)
+ goto exit;
+
+ ret = cdns2_ep_config(pep, enable);
+ if (ret) {
+ cdns2_free_tr_segment(pep);
+ ret = -EINVAL;
+ goto exit;
+ }
+
+ trace_cdns2_gadget_ep_enable(pep);
+
+ pep->ep_state &= ~(EP_STALLED | EP_STALL_PENDING);
+ pep->ep_state |= EP_ENABLED;
+ pep->wa1_set = 0;
+ pep->ring.enqueue = 0;
+ pep->ring.dequeue = 0;
+ reg = readl(&pdev->adma_regs->ep_sts);
+ pep->ring.pcs = !!DMA_EP_STS_CCS(reg);
+ pep->ring.ccs = !!DMA_EP_STS_CCS(reg);
+
+ writel(pep->ring.dma, &pdev->adma_regs->ep_traddr);
+
+ /* one TRB is reserved for link TRB used in DMULT mode*/
+ pep->ring.free_trbs = TRBS_PER_SEGMENT - 1;
+
+exit:
+ spin_unlock_irqrestore(&pdev->lock, flags);
+
+ return ret;
+}
+
+static int cdns2_gadget_ep_disable(struct usb_ep *ep)
+{
+ struct cdns2_endpoint *pep;
+ struct cdns2_request *preq;
+ struct cdns2_device *pdev;
+ unsigned long flags;
+ int val;
+
+ if (!ep)
+ return -EINVAL;
+
+ pep = ep_to_cdns2_ep(ep);
+ pdev = pep->pdev;
+
+ if (dev_WARN_ONCE(pdev->dev, !(pep->ep_state & EP_ENABLED),
+ "%s is already disabled\n", pep->name))
+ return 0;
+
+ spin_lock_irqsave(&pdev->lock, flags);
+
+ trace_cdns2_gadget_ep_disable(pep);
+
+ cdns2_select_ep(pdev, ep->desc->bEndpointAddress);
+
+ clear_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE);
+
+ /*
+ * Driver needs some time before resetting endpoint.
+ * It need waits for clearing DBUSY bit or for timeout expired.
+ * 10us is enough time for controller to stop transfer.
+ */
+ readl_poll_timeout_atomic(&pdev->adma_regs->ep_sts, val,
+ !(val & DMA_EP_STS_DBUSY), 1, 10);
+ writel(DMA_EP_CMD_EPRST, &pdev->adma_regs->ep_cmd);
+
+ readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val,
+ !(val & (DMA_EP_CMD_DFLUSH | DMA_EP_CMD_EPRST)),
+ 1, 1000);
+
+ while (!list_empty(&pep->pending_list)) {
+ preq = cdns2_next_preq(&pep->pending_list);
+ cdns2_gadget_giveback(pep, preq, -ESHUTDOWN);
+ }
+
+ while (!list_empty(&pep->deferred_list)) {
+ preq = cdns2_next_preq(&pep->deferred_list);
+ cdns2_gadget_giveback(pep, preq, -ESHUTDOWN);
+ }
+
+ ep->desc = NULL;
+ pep->ep_state &= ~EP_ENABLED;
+
+ spin_unlock_irqrestore(&pdev->lock, flags);
+
+ return 0;
+}
+
+static int cdns2_ep_enqueue(struct cdns2_endpoint *pep,
+ struct cdns2_request *preq,
+ gfp_t gfp_flags)
+{
+ struct cdns2_device *pdev = pep->pdev;
+ struct usb_request *request;
+ int ret;
+
+ request = &preq->request;
+ request->actual = 0;
+ request->status = -EINPROGRESS;
+
+ ret = usb_gadget_map_request_by_dev(pdev->dev, request, pep->dir);
+ if (ret) {
+ trace_cdns2_request_enqueue_error(preq);
+ return ret;
+ }
+
+ list_add_tail(&preq->list, &pep->deferred_list);
+ trace_cdns2_request_enqueue(preq);
+
+ if (!(pep->ep_state & EP_STALLED) && !(pep->ep_state & EP_STALL_PENDING))
+ cdns2_start_all_request(pdev, pep);
+
+ return 0;
+}
+
+static int cdns2_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
+ gfp_t gfp_flags)
+{
+ struct usb_request *zlp_request;
+ struct cdns2_request *preq;
+ struct cdns2_endpoint *pep;
+ struct cdns2_device *pdev;
+ unsigned long flags;
+ int ret;
+
+ if (!request || !ep)
+ return -EINVAL;
+
+ pep = ep_to_cdns2_ep(ep);
+ pdev = pep->pdev;
+
+ if (!(pep->ep_state & EP_ENABLED)) {
+ dev_err(pdev->dev, "%s: can't queue to disabled endpoint\n",
+ pep->name);
+ return -EINVAL;
+ }
+
+ spin_lock_irqsave(&pdev->lock, flags);
+
+ preq = to_cdns2_request(request);
+ ret = cdns2_ep_enqueue(pep, preq, gfp_flags);
+
+ if (ret == 0 && request->zero && request->length &&
+ (request->length % ep->maxpacket == 0)) {
+ struct cdns2_request *preq;
+
+ zlp_request = cdns2_gadget_ep_alloc_request(ep, GFP_ATOMIC);
+ zlp_request->buf = pdev->zlp_buf;
+ zlp_request->length = 0;
+
+ preq = to_cdns2_request(zlp_request);
+ ret = cdns2_ep_enqueue(pep, preq, gfp_flags);
+ }
+
+ spin_unlock_irqrestore(&pdev->lock, flags);
+ return ret;
+}
+
+int cdns2_gadget_ep_dequeue(struct usb_ep *ep,
+ struct usb_request *request)
+{
+ struct cdns2_request *preq, *preq_temp, *cur_preq;
+ struct cdns2_endpoint *pep;
+ struct cdns2_trb *link_trb;
+ u8 req_on_hw_ring = 0;
+ unsigned long flags;
+ u32 buffer;
+ int val, i;
+
+ if (!ep || !request || !ep->desc)
+ return -EINVAL;
+
+ pep = ep_to_cdns2_ep(ep);
+ if (!pep->endpoint.desc) {
+ dev_err(pep->pdev->dev, "%s: can't dequeue to disabled endpoint\n",
+ pep->name);
+ return -ESHUTDOWN;
+ }
+
+ /* Requests has been dequeued during disabling endpoint. */
+ if (!(pep->ep_state & EP_ENABLED))
+ return 0;
+
+ spin_lock_irqsave(&pep->pdev->lock, flags);
+
+ cur_preq = to_cdns2_request(request);
+ trace_cdns2_request_dequeue(cur_preq);
+
+ list_for_each_entry_safe(preq, preq_temp, &pep->pending_list, list) {
+ if (cur_preq == preq) {
+ req_on_hw_ring = 1;
+ goto found;
+ }
+ }
+
+ list_for_each_entry_safe(preq, preq_temp, &pep->deferred_list, list) {
+ if (cur_preq == preq)
+ goto found;
+ }
+
+ goto not_found;
+
+found:
+ link_trb = preq->trb;
+
+ /* Update ring only if removed request is on pending_req_list list. */
+ if (req_on_hw_ring && link_trb) {
+ /* Stop DMA */
+ writel(DMA_EP_CMD_DFLUSH, &pep->pdev->adma_regs->ep_cmd);
+
+ /* Wait for DFLUSH cleared. */
+ readl_poll_timeout_atomic(&pep->pdev->adma_regs->ep_cmd, val,
+ !(val & DMA_EP_CMD_DFLUSH), 1, 1000);
+
+ buffer = cpu_to_le32(TRB_BUFFER(pep->ring.dma +
+ ((preq->end_trb + 1) * TRB_SIZE)));
+
+ for (i = 0; i < preq->num_of_trb; i++) {
+ link_trb->buffer = buffer;
+ link_trb->control = cpu_to_le32((le32_to_cpu(link_trb->control)
+ & TRB_CYCLE) | TRB_CHAIN |
+ TRB_TYPE(TRB_LINK));
+
+ trace_cdns2_queue_trb(pep, link_trb);
+ link_trb = cdns2_next_trb(pep, link_trb);
+ }
+
+ if (pep->wa1_trb == preq->trb)
+ cdns2_wa1_restore_cycle_bit(pep);
+ }
+
+ cdns2_gadget_giveback(pep, cur_preq, -ECONNRESET);
+
+ preq = cdns2_next_preq(&pep->pending_list);
+ if (preq)
+ cdns2_rearm_transfer(pep, 1);
+
+not_found:
+ spin_unlock_irqrestore(&pep->pdev->lock, flags);
+ return 0;
+}
+
+int cdns2_halt_endpoint(struct cdns2_device *pdev,
+ struct cdns2_endpoint *pep,
+ int value)
+{
+ u8 __iomem *conf;
+ int dir = 0;
+
+ if (!(pep->ep_state & EP_ENABLED))
+ return -EPERM;
+
+ if (pep->dir) {
+ dir = ENDPRST_IO_TX;
+ conf = &pdev->epx_regs->ep[pep->num - 1].txcon;
+ } else {
+ conf = &pdev->epx_regs->ep[pep->num - 1].rxcon;
+ }
+
+ if (!value) {
+ struct cdns2_trb *trb = NULL;
+ struct cdns2_request *preq;
+ struct cdns2_trb trb_tmp;
+
+ preq = cdns2_next_preq(&pep->pending_list);
+ if (preq) {
+ trb = preq->trb;
+ if (trb) {
+ trb_tmp = *trb;
+ trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE);
+ }
+ }
+
+ trace_cdns2_ep_halt(pep, 0, 0);
+
+ /* Resets Sequence Number */
+ writeb(dir | pep->num, &pdev->epx_regs->endprst);
+ writeb(dir | ENDPRST_TOGRST | pep->num,
+ &pdev->epx_regs->endprst);
+
+ clear_reg_bit_8(conf, EPX_CON_STALL);
+
+ pep->ep_state &= ~(EP_STALLED | EP_STALL_PENDING);
+
+ if (preq) {
+ if (trb)
+ *trb = trb_tmp;
+
+ cdns2_rearm_transfer(pep, 1);
+ }
+
+ cdns2_start_all_request(pdev, pep);
+ } else {
+ trace_cdns2_ep_halt(pep, 1, 0);
+ set_reg_bit_8(conf, EPX_CON_STALL);
+ writeb(dir | pep->num, &pdev->epx_regs->endprst);
+ writeb(dir | ENDPRST_FIFORST | pep->num,
+ &pdev->epx_regs->endprst);
+ pep->ep_state |= EP_STALLED;
+ }
+
+ return 0;
+}
+
+/* Sets/clears stall on selected endpoint. */
+static int cdns2_gadget_ep_set_halt(struct usb_ep *ep, int value)
+{
+ struct cdns2_endpoint *pep = ep_to_cdns2_ep(ep);
+ struct cdns2_device *pdev = pep->pdev;
+ struct cdns2_request *preq;
+ unsigned long flags = 0;
+ int ret;
+
+ spin_lock_irqsave(&pdev->lock, flags);
+
+ preq = cdns2_next_preq(&pep->pending_list);
+ if (value && preq) {
+ trace_cdns2_ep_busy_try_halt_again(pep);
+ ret = -EAGAIN;
+ goto done;
+ }
+
+ if (!value)
+ pep->ep_state &= ~EP_WEDGE;
+
+ ret = cdns2_halt_endpoint(pdev, pep, value);
+
+done:
+ spin_unlock_irqrestore(&pdev->lock, flags);
+ return ret;
+}
+
+static int cdns2_gadget_ep_set_wedge(struct usb_ep *ep)
+{
+ struct cdns2_endpoint *pep = ep_to_cdns2_ep(ep);
+
+ cdns2_gadget_ep_set_halt(ep, 1);
+ pep->ep_state |= EP_WEDGE;
+
+ return 0;
+}
+
+static struct
+cdns2_endpoint *cdns2_find_available_ep(struct cdns2_device *pdev,
+ struct usb_endpoint_descriptor *desc)
+{
+ struct cdns2_endpoint *pep;
+ struct usb_ep *ep;
+ int ep_correct;
+
+ list_for_each_entry(ep, &pdev->gadget.ep_list, ep_list) {
+ unsigned long num;
+ int ret;
+ /* ep name pattern likes epXin or epXout. */
+ char c[2] = {ep->name[2], '\0'};
+
+ ret = kstrtoul(c, 10, &num);
+ if (ret)
+ return ERR_PTR(ret);
+ pep = ep_to_cdns2_ep(ep);
+
+ if (pep->num != num)
+ continue;
+
+ ep_correct = (pep->endpoint.caps.dir_in &&
+ usb_endpoint_dir_in(desc)) ||
+ (pep->endpoint.caps.dir_out &&
+ usb_endpoint_dir_out(desc));
+
+ if (ep_correct && !(pep->ep_state & EP_CLAIMED))
+ return pep;
+ }
+
+ return ERR_PTR(-ENOENT);
+}
+
+/*
+ * Function used to recognize which endpoints will be used to optimize
+ * on-chip memory usage.
+ */
+static struct
+usb_ep *cdns2_gadget_match_ep(struct usb_gadget *gadget,
+ struct usb_endpoint_descriptor *desc,
+ struct usb_ss_ep_comp_descriptor *comp_desc)
+{
+ struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
+ struct cdns2_endpoint *pep;
+ unsigned long flags;
+
+ pep = cdns2_find_available_ep(pdev, desc);
+ if (IS_ERR(pep)) {
+ dev_err(pdev->dev, "no available ep\n");
+ return NULL;
+ }
+
+ spin_lock_irqsave(&pdev->lock, flags);
+
+ if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC)
+ pep->buffering = 4;
+ else
+ pep->buffering = 1;
+
+ pep->ep_state |= EP_CLAIMED;
+ spin_unlock_irqrestore(&pdev->lock, flags);
+
+ return &pep->endpoint;
+}
+
+static const struct usb_ep_ops cdns2_gadget_ep_ops = {
+ .enable = cdns2_gadget_ep_enable,
+ .disable = cdns2_gadget_ep_disable,
+ .alloc_request = cdns2_gadget_ep_alloc_request,
+ .free_request = cdns2_gadget_ep_free_request,
+ .queue = cdns2_gadget_ep_queue,
+ .dequeue = cdns2_gadget_ep_dequeue,
+ .set_halt = cdns2_gadget_ep_set_halt,
+ .set_wedge = cdns2_gadget_ep_set_wedge,
+};
+
+static int cdns2_gadget_get_frame(struct usb_gadget *gadget)
+{
+ struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
+
+ return readw(&pdev->usb_regs->frmnr);
+}
+
+static int cdns2_gadget_wakeup(struct usb_gadget *gadget)
+{
+ struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
+ unsigned long flags;
+
+ spin_lock_irqsave(&pdev->lock, flags);
+ cdns2_wakeup(pdev);
+ spin_unlock_irqrestore(&pdev->lock, flags);
+
+ return 0;
+}
+
+static int cdns2_gadget_set_selfpowered(struct usb_gadget *gadget,
+ int is_selfpowered)
+{
+ struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
+ unsigned long flags;
+
+ spin_lock_irqsave(&pdev->lock, flags);
+ pdev->is_selfpowered = !!is_selfpowered;
+ spin_unlock_irqrestore(&pdev->lock, flags);
+ return 0;
+}
+
+/* Disable interrupts and begin the controller halting process. */
+static void cdns2_quiesce(struct cdns2_device *pdev)
+{
+ set_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_DISCON);
+
+ /* Disable interrupt. */
+ writeb(0, &pdev->interrupt_regs->extien),
+ writeb(0, &pdev->interrupt_regs->usbien),
+ writew(0, &pdev->adma_regs->ep_ien);
+
+ /* Clear interrupt line. */
+ writeb(0x0, &pdev->interrupt_regs->usbirq);
+}
+
+static void cdns2_gadget_config(struct cdns2_device *pdev)
+{
+ cdns2_ep0_config(pdev);
+
+ /* Enable DMA interrupts for all endpoints. */
+ writel(~0x0, &pdev->adma_regs->ep_ien);
+ cdns2_enable_l1(pdev, 0);
+ writeb(USB_IEN_INIT, &pdev->interrupt_regs->usbien);
+ writeb(EXTIRQ_WAKEUP, &pdev->interrupt_regs->extien);
+ writel(DMA_CONF_DMULT, &pdev->adma_regs->conf);
+}
+
+static int cdns2_gadget_pullup(struct usb_gadget *gadget, int is_on)
+{
+ struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
+ unsigned long flags;
+
+ trace_cdns2_pullup(is_on);
+
+ /*
+ * Disable events handling while controller is being
+ * enabled/disabled.
+ */
+ disable_irq(pdev->irq);
+ spin_lock_irqsave(&pdev->lock, flags);
+
+ if (is_on) {
+ cdns2_gadget_config(pdev);
+ clear_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_DISCON);
+ } else {
+ cdns2_quiesce(pdev);
+ }
+
+ spin_unlock_irqrestore(&pdev->lock, flags);
+ enable_irq(pdev->irq);
+
+ return 0;
+}
+
+static int cdns2_gadget_udc_start(struct usb_gadget *gadget,
+ struct usb_gadget_driver *driver)
+{
+ struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
+ enum usb_device_speed max_speed = driver->max_speed;
+ unsigned long flags;
+
+ spin_lock_irqsave(&pdev->lock, flags);
+ pdev->gadget_driver = driver;
+
+ /* Limit speed if necessary. */
+ max_speed = min(driver->max_speed, gadget->max_speed);
+
+ switch (max_speed) {
+ case USB_SPEED_FULL:
+ writeb(SPEEDCTRL_HSDISABLE, &pdev->usb_regs->speedctrl);
+ break;
+ case USB_SPEED_HIGH:
+ writeb(0, &pdev->usb_regs->speedctrl);
+ break;
+ default:
+ dev_err(pdev->dev, "invalid maximum_speed parameter %d\n",
+ max_speed);
+ fallthrough;
+ case USB_SPEED_UNKNOWN:
+ /* Default to highspeed. */
+ max_speed = USB_SPEED_HIGH;
+ break;
+ }
+
+ /* Reset all USB endpoints. */
+ writeb(ENDPRST_IO_TX, &pdev->usb_regs->endprst);
+ writeb(ENDPRST_FIFORST | ENDPRST_TOGRST | ENDPRST_IO_TX,
+ &pdev->usb_regs->endprst);
+ writeb(ENDPRST_FIFORST | ENDPRST_TOGRST, &pdev->usb_regs->endprst);
+
+ cdns2_eps_onchip_buffer_init(pdev);
+
+ cdns2_gadget_config(pdev);
+ spin_unlock_irqrestore(&pdev->lock, flags);
+
+ return 0;
+}
+
+static int cdns2_gadget_udc_stop(struct usb_gadget *gadget)
+{
+ struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
+ struct cdns2_endpoint *pep;
+ u32 bEndpointAddress;
+ struct usb_ep *ep;
+ int val;
+
+ pdev->gadget_driver = NULL;
+ pdev->gadget.speed = USB_SPEED_UNKNOWN;
+
+ list_for_each_entry(ep, &pdev->gadget.ep_list, ep_list) {
+ pep = ep_to_cdns2_ep(ep);
+ bEndpointAddress = pep->num | pep->dir;
+ cdns2_select_ep(pdev, bEndpointAddress);
+ writel(DMA_EP_CMD_EPRST, &pdev->adma_regs->ep_cmd);
+ readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val,
+ !(val & DMA_EP_CMD_EPRST), 1, 100);
+ }
+
+ cdns2_quiesce(pdev);
+
+ writeb(ENDPRST_IO_TX, &pdev->usb_regs->endprst);
+ writeb(ENDPRST_FIFORST | ENDPRST_TOGRST | ENDPRST_IO_TX,
+ &pdev->epx_regs->endprst);
+ writeb(ENDPRST_FIFORST | ENDPRST_TOGRST, &pdev->epx_regs->endprst);
+
+ return 0;
+}
+
+static const struct usb_gadget_ops cdns2_gadget_ops = {
+ .get_frame = cdns2_gadget_get_frame,
+ .wakeup = cdns2_gadget_wakeup,
+ .set_selfpowered = cdns2_gadget_set_selfpowered,
+ .pullup = cdns2_gadget_pullup,
+ .udc_start = cdns2_gadget_udc_start,
+ .udc_stop = cdns2_gadget_udc_stop,
+ .match_ep = cdns2_gadget_match_ep,
+};
+
+static void cdns2_free_all_eps(struct cdns2_device *pdev)
+{
+ int i;
+
+ for (i = 0; i < CDNS2_ENDPOINTS_NUM; i++)
+ cdns2_free_tr_segment(&pdev->eps[i]);
+}
+
+/* Initializes software endpoints of gadget. */
+static int cdns2_init_eps(struct cdns2_device *pdev)
+{
+ struct cdns2_endpoint *pep;
+ int i;
+
+ for (i = 0; i < CDNS2_ENDPOINTS_NUM; i++) {
+ bool direction = !(i & 1); /* Start from OUT endpoint. */
+ u8 epnum = ((i + 1) >> 1);
+
+ /*
+ * Endpoints are being held in pdev->eps[] in form:
+ * ep0, ep1out, ep1in ... ep15out, ep15in.
+ */
+ if (!CDNS2_IF_EP_EXIST(pdev, epnum, direction))
+ continue;
+
+ pep = &pdev->eps[i];
+ pep->pdev = pdev;
+ pep->num = epnum;
+ /* 0 for OUT, 1 for IN. */
+ pep->dir = direction ? USB_DIR_IN : USB_DIR_OUT;
+ pep->idx = i;
+
+ /* Ep0in and ep0out are represented by pdev->eps[0]. */
+ if (!epnum) {
+ int ret;
+
+ snprintf(pep->name, sizeof(pep->name), "ep%d%s",
+ epnum, "BiDir");
+
+ cdns2_init_ep0(pdev, pep);
+
+ ret = cdns2_alloc_tr_segment(pep);
+ if (ret) {
+ dev_err(pdev->dev, "Failed to init ep0\n");
+ return ret;
+ }
+ } else {
+ snprintf(pep->name, sizeof(pep->name), "ep%d%s",
+ epnum, !!direction ? "in" : "out");
+ pep->endpoint.name = pep->name;
+
+ usb_ep_set_maxpacket_limit(&pep->endpoint, 1024);
+ pep->endpoint.ops = &cdns2_gadget_ep_ops;
+ list_add_tail(&pep->endpoint.ep_list, &pdev->gadget.ep_list);
+
+ pep->endpoint.caps.dir_in = direction;
+ pep->endpoint.caps.dir_out = !direction;
+
+ pep->endpoint.caps.type_iso = 1;
+ pep->endpoint.caps.type_bulk = 1;
+ pep->endpoint.caps.type_int = 1;
+ }
+
+ pep->endpoint.name = pep->name;
+ pep->ep_state = 0;
+
+ dev_dbg(pdev->dev, "Init %s, SupType: CTRL: %s, INT: %s, "
+ "BULK: %s, ISOC %s, SupDir IN: %s, OUT: %s\n",
+ pep->name,
+ (pep->endpoint.caps.type_control) ? "yes" : "no",
+ (pep->endpoint.caps.type_int) ? "yes" : "no",
+ (pep->endpoint.caps.type_bulk) ? "yes" : "no",
+ (pep->endpoint.caps.type_iso) ? "yes" : "no",
+ (pep->endpoint.caps.dir_in) ? "yes" : "no",
+ (pep->endpoint.caps.dir_out) ? "yes" : "no");
+
+ INIT_LIST_HEAD(&pep->pending_list);
+ INIT_LIST_HEAD(&pep->deferred_list);
+ }
+
+ return 0;
+}
+
+static int cdns2_gadget_start(struct cdns2_device *pdev)
+{
+ u32 max_speed;
+ void *buf;
+ int val;
+ int ret;
+
+ pdev->usb_regs = pdev->regs;
+ pdev->ep0_regs = pdev->regs;
+ pdev->epx_regs = pdev->regs;
+ pdev->interrupt_regs = pdev->regs;
+ pdev->adma_regs = pdev->regs + CDNS2_ADMA_REGS_OFFSET;
+
+ /* Reset controller. */
+ set_reg_bit_8(&pdev->usb_regs->cpuctrl, CPUCTRL_SW_RST);
+
+ ret = readl_poll_timeout_atomic(&pdev->usb_regs->cpuctrl, val,
+ !(val & CPUCTRL_SW_RST), 1, 10000);
+ if (ret) {
+ dev_err(pdev->dev, "Error: reset controller timeout\n");
+ return -EINVAL;
+ }
+
+ usb_initialize_gadget(pdev->dev, &pdev->gadget, NULL);
+
+ device_property_read_u16(pdev->dev, "cdns,on-chip-tx-buff-size",
+ &pdev->onchip_tx_buf);
+ device_property_read_u16(pdev->dev, "cdns,on-chip-rx-buff-size",
+ &pdev->onchip_rx_buf);
+ device_property_read_u32(pdev->dev, "cdns,avail-endpoints",
+ &pdev->eps_supported);
+
+ /*
+ * Driver assumes that each USBHS controller has at least
+ * one IN and one OUT non control endpoint.
+ */
+ if (!pdev->onchip_tx_buf && !pdev->onchip_rx_buf) {
+ ret = -EINVAL;
+ dev_err(pdev->dev, "Invalid on-chip memory configuration\n");
+ goto put_gadget;
+ }
+
+ if (!(pdev->eps_supported & ~0x00010001)) {
+ ret = -EINVAL;
+ dev_err(pdev->dev, "No hardware endpoints available\n");
+ goto put_gadget;
+ }
+
+ max_speed = usb_get_maximum_speed(pdev->dev);
+
+ switch (max_speed) {
+ case USB_SPEED_FULL:
+ case USB_SPEED_HIGH:
+ break;
+ default:
+ dev_err(pdev->dev, "invalid maximum_speed parameter %d\n",
+ max_speed);
+ fallthrough;
+ case USB_SPEED_UNKNOWN:
+ max_speed = USB_SPEED_HIGH;
+ break;
+ }
+
+ pdev->gadget.max_speed = max_speed;
+ pdev->gadget.speed = USB_SPEED_UNKNOWN;
+ pdev->gadget.ops = &cdns2_gadget_ops;
+ pdev->gadget.name = "usbhs-gadget";
+ pdev->gadget.quirk_avoids_skb_reserve = 1;
+ pdev->gadget.irq = pdev->irq;
+
+ spin_lock_init(&pdev->lock);
+ INIT_WORK(&pdev->pending_status_wq, cdns2_pending_setup_status_handler);
+
+ /* Initialize endpoint container. */
+ INIT_LIST_HEAD(&pdev->gadget.ep_list);
+ pdev->eps_dma_pool = dma_pool_create("cdns2_eps_dma_pool", pdev->dev,
+ TR_SEG_SIZE, 8, 0);
+ if (!pdev->eps_dma_pool) {
+ dev_err(pdev->dev, "Failed to create TRB dma pool\n");
+ ret = -ENOMEM;
+ goto put_gadget;
+ }
+
+ ret = cdns2_init_eps(pdev);
+ if (ret) {
+ dev_err(pdev->dev, "Failed to create endpoints\n");
+ goto destroy_dma_pool;
+ }
+
+ pdev->gadget.sg_supported = 1;
+
+ pdev->zlp_buf = kzalloc(CDNS2_EP_ZLP_BUF_SIZE, GFP_KERNEL);
+ if (!pdev->zlp_buf) {
+ ret = -ENOMEM;
+ goto destroy_dma_pool;
+ }
+
+ /* Allocate memory for setup packet buffer. */
+ buf = dma_alloc_coherent(pdev->dev, 8, &pdev->ep0_preq.request.dma,
+ GFP_DMA);
+ pdev->ep0_preq.request.buf = buf;
+
+ if (!pdev->ep0_preq.request.buf) {
+ ret = -ENOMEM;
+ goto free_zlp_buf;
+ }
+
+ /* Add USB gadget device. */
+ ret = usb_add_gadget(&pdev->gadget);
+ if (ret < 0) {
+ dev_err(pdev->dev, "Failed to add gadget\n");
+ goto free_ep0_buf;
+ }
+
+ return 0;
+
+free_ep0_buf:
+ dma_free_coherent(pdev->dev, 8, pdev->ep0_preq.request.buf,
+ pdev->ep0_preq.request.dma);
+free_zlp_buf:
+ kfree(pdev->zlp_buf);
+destroy_dma_pool:
+ dma_pool_destroy(pdev->eps_dma_pool);
+put_gadget:
+ usb_put_gadget(&pdev->gadget);
+
+ return ret;
+}
+
+int cdns2_gadget_suspend(struct cdns2_device *pdev)
+{
+ unsigned long flags;
+
+ cdns2_disconnect_gadget(pdev);
+
+ spin_lock_irqsave(&pdev->lock, flags);
+ pdev->gadget.speed = USB_SPEED_UNKNOWN;
+
+ trace_cdns2_device_state("notattached");
+ usb_gadget_set_state(&pdev->gadget, USB_STATE_NOTATTACHED);
+ cdns2_enable_l1(pdev, 0);
+
+ /* Disable interrupt for device. */
+ writeb(0, &pdev->interrupt_regs->usbien);
+ writel(0, &pdev->adma_regs->ep_ien);
+ spin_unlock_irqrestore(&pdev->lock, flags);
+
+ return 0;
+}
+
+int cdns2_gadget_resume(struct cdns2_device *pdev, bool hibernated)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&pdev->lock, flags);
+
+ if (!pdev->gadget_driver) {
+ spin_unlock_irqrestore(&pdev->lock, flags);
+ return 0;
+ }
+
+ cdns2_gadget_config(pdev);
+
+ if (hibernated)
+ clear_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_DISCON);
+
+ spin_unlock_irqrestore(&pdev->lock, flags);
+
+ return 0;
+}
+
+void cdns2_gadget_remove(struct cdns2_device *pdev)
+{
+ pm_runtime_mark_last_busy(pdev->dev);
+ pm_runtime_put_autosuspend(pdev->dev);
+
+ usb_del_gadget(&pdev->gadget);
+ cdns2_free_all_eps(pdev);
+
+ dma_pool_destroy(pdev->eps_dma_pool);
+ kfree(pdev->zlp_buf);
+ usb_put_gadget(&pdev->gadget);
+}
+
+int cdns2_gadget_init(struct cdns2_device *pdev)
+{
+ int ret;
+
+ /* Ensure 32-bit DMA Mask. */
+ ret = dma_set_mask_and_coherent(pdev->dev, DMA_BIT_MASK(32));
+ if (ret) {
+ dev_err(pdev->dev, "Failed to set dma mask: %d\n", ret);
+ return ret;
+ }
+
+ pm_runtime_get_sync(pdev->dev);
+
+ cdsn2_isoc_burst_opt(pdev);
+
+ ret = cdns2_gadget_start(pdev);
+ if (ret) {
+ pm_runtime_put_sync(pdev->dev);
+ return ret;
+ }
+
+ /*
+ * Because interrupt line can be shared with other components in
+ * driver it can't use IRQF_ONESHOT flag here.
+ */
+ ret = devm_request_threaded_irq(pdev->dev, pdev->irq,
+ cdns2_usb_irq_handler,
+ cdns2_thread_irq_handler,
+ IRQF_SHARED,
+ dev_name(pdev->dev),
+ pdev);
+ if (ret)
+ goto err0;
+
+ return 0;
+
+err0:
+ cdns2_gadget_remove(pdev);
+
+ return ret;
+}
diff --git a/drivers/usb/gadget/udc/cdns2/cdns2-gadget.h b/drivers/usb/gadget/udc/cdns2/cdns2-gadget.h
new file mode 100644
index 0000000000..71e2f62d65
--- /dev/null
+++ b/drivers/usb/gadget/udc/cdns2/cdns2-gadget.h
@@ -0,0 +1,707 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * USBHS-DEV device controller driver header file
+ *
+ * Copyright (C) 2023 Cadence.
+ *
+ * Author: Pawel Laszczak <pawell@cadence.com>
+ */
+
+#ifndef __LINUX_CDNS2_GADGET
+#define __LINUX_CDNS2_GADGET
+
+#include <linux/usb/gadget.h>
+#include <linux/dma-direction.h>
+
+/*
+ * USBHS register interface.
+ * This corresponds to the USBHS Device Controller Interface.
+ */
+
+/**
+ * struct cdns2_ep0_regs - endpoint 0 related registers.
+ * @rxbc: receive (OUT) 0 endpoint byte count register.
+ * @txbc: transmit (IN) 0 endpoint byte count register.
+ * @cs: 0 endpoint control and status register.
+ * @reserved1: reserved.
+ * @fifo: 0 endpoint fifo register.
+ * @reserved2: reserved.
+ * @setupdat: SETUP data register.
+ * @reserved4: reserved.
+ * @maxpack: 0 endpoint max packet size.
+ */
+struct cdns2_ep0_regs {
+ __u8 rxbc;
+ __u8 txbc;
+ __u8 cs;
+ __u8 reserved1[4];
+ __u8 fifo;
+ __le32 reserved2[94];
+ __u8 setupdat[8];
+ __u8 reserved4[88];
+ __u8 maxpack;
+} __packed __aligned(4);
+
+/* EP0CS - bitmasks. */
+/* Endpoint 0 stall bit for status stage. */
+#define EP0CS_STALL BIT(0)
+/* HSNAK bit. */
+#define EP0CS_HSNAK BIT(1)
+/* IN 0 endpoint busy bit. */
+#define EP0CS_TXBSY_MSK BIT(2)
+/* OUT 0 endpoint busy bit. */
+#define EP0CS_RXBSY_MSK BIT(3)
+/* Send STALL in the data stage phase. */
+#define EP0CS_DSTALL BIT(4)
+/* SETUP buffer content was changed. */
+#define EP0CS_CHGSET BIT(7)
+
+/* EP0FIFO - bitmasks. */
+/* Direction. */
+#define EP0_FIFO_IO_TX BIT(4)
+/* FIFO auto bit. */
+#define EP0_FIFO_AUTO BIT(5)
+/* FIFO commit bit. */
+#define EP0_FIFO_COMMIT BIT(6)
+/* FIFO access bit. */
+#define EP0_FIFO_ACCES BIT(7)
+
+/**
+ * struct cdns2_epx_base - base endpoint registers.
+ * @rxbc: OUT endpoint byte count register.
+ * @rxcon: OUT endpoint control register.
+ * @rxcs: OUT endpoint control and status register.
+ * @txbc: IN endpoint byte count register.
+ * @txcon: IN endpoint control register.
+ * @txcs: IN endpoint control and status register.
+ */
+struct cdns2_epx_base {
+ __le16 rxbc;
+ __u8 rxcon;
+ __u8 rxcs;
+ __le16 txbc;
+ __u8 txcon;
+ __u8 txcs;
+} __packed __aligned(4);
+
+/* rxcon/txcon - endpoint control register bitmasks. */
+/* Endpoint buffering: 0 - single buffering ... 3 - quad buffering. */
+#define EPX_CON_BUF GENMASK(1, 0)
+/* Endpoint type. */
+#define EPX_CON_TYPE GENMASK(3, 2)
+/* Endpoint type: isochronous. */
+#define EPX_CON_TYPE_ISOC 0x4
+/* Endpoint type: bulk. */
+#define EPX_CON_TYPE_BULK 0x8
+/* Endpoint type: interrupt. */
+#define EPX_CON_TYPE_INT 0xC
+/* Number of packets per microframe. */
+#define EPX_CON_ISOD GENMASK(5, 4)
+#define EPX_CON_ISOD_SHIFT 0x4
+/* Endpoint stall bit. */
+#define EPX_CON_STALL BIT(6)
+/* Endpoint enable bit.*/
+#define EPX_CON_VAL BIT(7)
+
+/* rxcs/txcs - endpoint control and status bitmasks. */
+/* Data sequence error for the ISO endpoint. */
+#define EPX_CS_ERR(p) ((p) & BIT(0))
+
+/**
+ * struct cdns2_epx_regs - endpoint 1..15 related registers.
+ * @reserved: reserved.
+ * @ep: none control endpoints array.
+ * @reserved2: reserved.
+ * @endprst: endpoint reset register.
+ * @reserved3: reserved.
+ * @isoautoarm: ISO auto-arm register.
+ * @reserved4: reserved.
+ * @isodctrl: ISO control register.
+ * @reserved5: reserved.
+ * @isoautodump: ISO auto dump enable register.
+ * @reserved6: reserved.
+ * @rxmaxpack: receive (OUT) Max packet size register.
+ * @reserved7: reserved.
+ * @rxstaddr: receive (OUT) start address endpoint buffer register.
+ * @reserved8: reserved.
+ * @txstaddr: transmit (IN) start address endpoint buffer register.
+ * @reserved9: reserved.
+ * @txmaxpack: transmit (IN) Max packet size register.
+ */
+struct cdns2_epx_regs {
+ __le32 reserved[2];
+ struct cdns2_epx_base ep[15];
+ __u8 reserved2[290];
+ __u8 endprst;
+ __u8 reserved3[41];
+ __le16 isoautoarm;
+ __u8 reserved4[10];
+ __le16 isodctrl;
+ __le16 reserved5;
+ __le16 isoautodump;
+ __le32 reserved6;
+ __le16 rxmaxpack[15];
+ __le32 reserved7[65];
+ __le32 rxstaddr[15];
+ __u8 reserved8[4];
+ __le32 txstaddr[15];
+ __u8 reserved9[98];
+ __le16 txmaxpack[15];
+} __packed __aligned(4);
+
+/* ENDPRST - bitmasks. */
+/* Endpoint number. */
+#define ENDPRST_EP GENMASK(3, 0)
+/* IN direction bit. */
+#define ENDPRST_IO_TX BIT(4)
+/* Toggle reset bit. */
+#define ENDPRST_TOGRST BIT(5)
+/* FIFO reset bit. */
+#define ENDPRST_FIFORST BIT(6)
+/* Toggle status and reset bit. */
+#define ENDPRST_TOGSETQ BIT(7)
+
+/**
+ * struct cdns2_interrupt_regs - USB interrupt related registers.
+ * @reserved: reserved.
+ * @usbirq: USB interrupt request register.
+ * @extirq: external interrupt request register.
+ * @rxpngirq: external interrupt request register.
+ * @reserved1: reserved.
+ * @usbien: USB interrupt enable register.
+ * @extien: external interrupt enable register.
+ * @reserved2: reserved.
+ * @usbivect: USB interrupt vector register.
+ */
+struct cdns2_interrupt_regs {
+ __u8 reserved[396];
+ __u8 usbirq;
+ __u8 extirq;
+ __le16 rxpngirq;
+ __le16 reserved1[4];
+ __u8 usbien;
+ __u8 extien;
+ __le16 reserved2[3];
+ __u8 usbivect;
+} __packed __aligned(4);
+
+/* EXTIRQ and EXTIEN - bitmasks. */
+/* VBUS fault fall interrupt. */
+#define EXTIRQ_VBUSFAULT_FALL BIT(0)
+/* VBUS fault fall interrupt. */
+#define EXTIRQ_VBUSFAULT_RISE BIT(1)
+/* Wake up interrupt bit. */
+#define EXTIRQ_WAKEUP BIT(7)
+
+/* USBIEN and USBIRQ - bitmasks. */
+/* SETUP data valid interrupt bit.*/
+#define USBIRQ_SUDAV BIT(0)
+/* Start-of-frame interrupt bit. */
+#define USBIRQ_SOF BIT(1)
+/* SETUP token interrupt bit. */
+#define USBIRQ_SUTOK BIT(2)
+/* USB suspend interrupt bit. */
+#define USBIRQ_SUSPEND BIT(3)
+/* USB reset interrupt bit. */
+#define USBIRQ_URESET BIT(4)
+/* USB high-speed mode interrupt bit. */
+#define USBIRQ_HSPEED BIT(5)
+/* Link Power Management interrupt bit. */
+#define USBIRQ_LPM BIT(7)
+
+#define USB_IEN_INIT (USBIRQ_SUDAV | USBIRQ_SUSPEND | USBIRQ_URESET \
+ | USBIRQ_HSPEED | USBIRQ_LPM)
+/**
+ * struct cdns2_usb_regs - USB controller registers.
+ * @reserved: reserved.
+ * @lpmctrl: LPM control register.
+ * @lpmclock: LPM clock register.
+ * @reserved2: reserved.
+ * @endprst: endpoint reset register.
+ * @usbcs: USB control and status register.
+ * @frmnr: USB frame counter register.
+ * @fnaddr: function Address register.
+ * @clkgate: clock gate register.
+ * @fifoctrl: FIFO control register.
+ * @speedctrl: speed Control register.
+ * @sleep_clkgate: sleep Clock Gate register.
+ * @reserved3: reserved.
+ * @cpuctrl: microprocessor control register.
+ */
+struct cdns2_usb_regs {
+ __u8 reserved[4];
+ __u16 lpmctrl;
+ __u8 lpmclock;
+ __u8 reserved2[411];
+ __u8 endprst;
+ __u8 usbcs;
+ __le16 frmnr;
+ __u8 fnaddr;
+ __u8 clkgate;
+ __u8 fifoctrl;
+ __u8 speedctrl;
+ __u8 sleep_clkgate;
+ __u8 reserved3[533];
+ __u8 cpuctrl;
+} __packed __aligned(4);
+
+/* LPMCTRL - bitmasks. */
+/* BESL (Best Effort Service Latency). */
+#define LPMCTRLLL_HIRD GENMASK(7, 4)
+/* Last received Remote Wakeup field from LPM Extended Token packet. */
+#define LPMCTRLLH_BREMOTEWAKEUP BIT(8)
+/* Reflects value of the lpmnyet bit located in the usbcs[1] register. */
+#define LPMCTRLLH_LPMNYET BIT(16)
+
+/* LPMCLOCK - bitmasks. */
+/*
+ * If bit is 1 the controller automatically turns off clock
+ * (utmisleepm goes to low), else the microprocessor should use
+ * sleep clock gate register to turn off clock.
+ */
+#define LPMCLOCK_SLEEP_ENTRY BIT(7)
+
+/* USBCS - bitmasks. */
+/* Send NYET handshake for the LPM transaction. */
+#define USBCS_LPMNYET BIT(2)
+/* Remote wake-up bit. */
+#define USBCS_SIGRSUME BIT(5)
+/* Software disconnect bit. */
+#define USBCS_DISCON BIT(6)
+/* Indicates that a wakeup pin resumed the controller. */
+#define USBCS_WAKESRC BIT(7)
+
+/* FIFOCTRL - bitmasks. */
+/* Endpoint number. */
+#define FIFOCTRL_EP GENMASK(3, 0)
+/* Direction bit. */
+#define FIFOCTRL_IO_TX BIT(4)
+/* FIFO auto bit. */
+#define FIFOCTRL_FIFOAUTO BIT(5)
+/* FIFO commit bit. */
+#define FIFOCTRL_FIFOCMIT BIT(6)
+/* FIFO access bit. */
+#define FIFOCTRL_FIFOACC BIT(7)
+
+/* SPEEDCTRL - bitmasks. */
+/* Device works in Full Speed. */
+#define SPEEDCTRL_FS BIT(1)
+/* Device works in High Speed. */
+#define SPEEDCTRL_HS BIT(2)
+/* Force FS mode. */
+#define SPEEDCTRL_HSDISABLE BIT(7)
+
+/* CPUCTRL- bitmasks. */
+/* Controller reset bit. */
+#define CPUCTRL_SW_RST BIT(1)
+
+/**
+ * struct cdns2_adma_regs - ADMA controller registers.
+ * @conf: DMA global configuration register.
+ * @sts: DMA global Status register.
+ * @reserved1: reserved.
+ * @ep_sel: DMA endpoint select register.
+ * @ep_traddr: DMA endpoint transfer ring address register.
+ * @ep_cfg: DMA endpoint configuration register.
+ * @ep_cmd: DMA endpoint command register.
+ * @ep_sts: DMA endpoint status register.
+ * @reserved2: reserved.
+ * @ep_sts_en: DMA endpoint status enable register.
+ * @drbl: DMA doorbell register.
+ * @ep_ien: DMA endpoint interrupt enable register.
+ * @ep_ists: DMA endpoint interrupt status register.
+ * @axim_ctrl: AXI Master Control register.
+ * @axim_id: AXI Master ID register.
+ * @reserved3: reserved.
+ * @axim_cap: AXI Master Wrapper Extended Capability.
+ * @reserved4: reserved.
+ * @axim_ctrl0: AXI Master Wrapper Extended Capability Control Register 0.
+ * @axim_ctrl1: AXI Master Wrapper Extended Capability Control Register 1.
+ */
+struct cdns2_adma_regs {
+ __le32 conf;
+ __le32 sts;
+ __le32 reserved1[5];
+ __le32 ep_sel;
+ __le32 ep_traddr;
+ __le32 ep_cfg;
+ __le32 ep_cmd;
+ __le32 ep_sts;
+ __le32 reserved2;
+ __le32 ep_sts_en;
+ __le32 drbl;
+ __le32 ep_ien;
+ __le32 ep_ists;
+ __le32 axim_ctrl;
+ __le32 axim_id;
+ __le32 reserved3;
+ __le32 axim_cap;
+ __le32 reserved4;
+ __le32 axim_ctrl0;
+ __le32 axim_ctrl1;
+};
+
+#define CDNS2_ADMA_REGS_OFFSET 0x400
+
+/* DMA_CONF - bitmasks. */
+/* Reset USB device configuration. */
+#define DMA_CONF_CFGRST BIT(0)
+/* Singular DMA transfer mode.*/
+#define DMA_CONF_DSING BIT(8)
+/* Multiple DMA transfers mode.*/
+#define DMA_CONF_DMULT BIT(9)
+
+/* DMA_EP_CFG - bitmasks. */
+/* Endpoint enable. */
+#define DMA_EP_CFG_ENABLE BIT(0)
+
+/* DMA_EP_CMD - bitmasks. */
+/* Endpoint reset. */
+#define DMA_EP_CMD_EPRST BIT(0)
+/* Transfer descriptor ready. */
+#define DMA_EP_CMD_DRDY BIT(6)
+/* Data flush. */
+#define DMA_EP_CMD_DFLUSH BIT(7)
+
+/* DMA_EP_STS - bitmasks. */
+/* Interrupt On Complete. */
+#define DMA_EP_STS_IOC BIT(2)
+/* Interrupt on Short Packet. */
+#define DMA_EP_STS_ISP BIT(3)
+/* Transfer descriptor missing. */
+#define DMA_EP_STS_DESCMIS BIT(4)
+/* TRB error. */
+#define DMA_EP_STS_TRBERR BIT(7)
+/* DMA busy bit. */
+#define DMA_EP_STS_DBUSY BIT(9)
+/* Current Cycle Status. */
+#define DMA_EP_STS_CCS(p) ((p) & BIT(11))
+/* OUT size mismatch. */
+#define DMA_EP_STS_OUTSMM BIT(14)
+/* ISO transmission error. */
+#define DMA_EP_STS_ISOERR BIT(15)
+
+/* DMA_EP_STS_EN - bitmasks. */
+/* OUT transfer missing descriptor enable. */
+#define DMA_EP_STS_EN_DESCMISEN BIT(4)
+/* TRB enable. */
+#define DMA_EP_STS_EN_TRBERREN BIT(7)
+/* OUT size mismatch enable. */
+#define DMA_EP_STS_EN_OUTSMMEN BIT(14)
+/* ISO transmission error enable. */
+#define DMA_EP_STS_EN_ISOERREN BIT(15)
+
+/* DMA_EP_IEN - bitmasks. */
+#define DMA_EP_IEN(index) (1 << (index))
+#define DMA_EP_IEN_EP_OUT0 BIT(0)
+#define DMA_EP_IEN_EP_IN0 BIT(16)
+
+/* DMA_EP_ISTS - bitmasks. */
+#define DMA_EP_ISTS(index) (1 << (index))
+#define DMA_EP_ISTS_EP_OUT0 BIT(0)
+#define DMA_EP_ISTS_EP_IN0 BIT(16)
+
+#define gadget_to_cdns2_device(g) (container_of(g, struct cdns2_device, gadget))
+#define ep_to_cdns2_ep(ep) (container_of(ep, struct cdns2_endpoint, endpoint))
+
+/*-------------------------------------------------------------------------*/
+#define TRBS_PER_SEGMENT 600
+#define ISO_MAX_INTERVAL 8
+#define MAX_TRB_LENGTH BIT(16)
+#define MAX_ISO_SIZE 3076
+/*
+ * To improve performance the TRB buffer pointers can't cross
+ * 4KB boundaries.
+ */
+#define TRB_MAX_ISO_BUFF_SHIFT 12
+#define TRB_MAX_ISO_BUFF_SIZE BIT(TRB_MAX_ISO_BUFF_SHIFT)
+/* How much data is left before the 4KB boundary? */
+#define TRB_BUFF_LEN_UP_TO_BOUNDARY(addr) (TRB_MAX_ISO_BUFF_SIZE - \
+ ((addr) & (TRB_MAX_ISO_BUFF_SIZE - 1)))
+
+#if TRBS_PER_SEGMENT < 2
+#error "Incorrect TRBS_PER_SEGMENT. Minimal Transfer Ring size is 2."
+#endif
+
+/**
+ * struct cdns2_trb - represent Transfer Descriptor block.
+ * @buffer: pointer to buffer data.
+ * @length: length of data.
+ * @control: control flags.
+ *
+ * This structure describes transfer block handled by DMA module.
+ */
+struct cdns2_trb {
+ __le32 buffer;
+ __le32 length;
+ __le32 control;
+};
+
+#define TRB_SIZE (sizeof(struct cdns2_trb))
+/*
+ * These two extra TRBs are reserved for isochronous transfer
+ * to inject 0 length packet and extra LINK TRB to synchronize the ISO transfer.
+ */
+#define TRB_ISO_RESERVED 2
+#define TR_SEG_SIZE (TRB_SIZE * (TRBS_PER_SEGMENT + TRB_ISO_RESERVED))
+
+/* TRB bit mask. */
+#define TRB_TYPE_BITMASK GENMASK(15, 10)
+#define TRB_TYPE(p) ((p) << 10)
+#define TRB_FIELD_TO_TYPE(p) (((p) & TRB_TYPE_BITMASK) >> 10)
+
+/* TRB type IDs. */
+/* Used for Bulk, Interrupt, ISOC, and control data stage. */
+#define TRB_NORMAL 1
+/* TRB for linking ring segments. */
+#define TRB_LINK 6
+
+/* Cycle bit - indicates TRB ownership by driver or hw. */
+#define TRB_CYCLE BIT(0)
+/*
+ * When set to '1', the device will toggle its interpretation of the Cycle bit.
+ */
+#define TRB_TOGGLE BIT(1)
+/* Interrupt on short packet. */
+#define TRB_ISP BIT(2)
+/* Chain bit associate this TRB with next one TRB. */
+#define TRB_CHAIN BIT(4)
+/* Interrupt on completion. */
+#define TRB_IOC BIT(5)
+
+/* Transfer_len bitmasks. */
+#define TRB_LEN(p) ((p) & GENMASK(16, 0))
+#define TRB_BURST(p) (((p) << 24) & GENMASK(31, 24))
+#define TRB_FIELD_TO_BURST(p) (((p) & GENMASK(31, 24)) >> 24)
+
+/* Data buffer pointer bitmasks. */
+#define TRB_BUFFER(p) ((p) & GENMASK(31, 0))
+
+/*-------------------------------------------------------------------------*/
+/* Driver numeric constants. */
+
+/* Maximum address that can be assigned to device. */
+#define USB_DEVICE_MAX_ADDRESS 127
+
+/* One control and 15 IN and 15 OUT endpoints. */
+#define CDNS2_ENDPOINTS_NUM 31
+
+#define CDNS2_EP_ZLP_BUF_SIZE 512
+
+/*-------------------------------------------------------------------------*/
+/* Used structures. */
+
+struct cdns2_device;
+
+/**
+ * struct cdns2_ring - transfer ring representation.
+ * @trbs: pointer to transfer ring.
+ * @dma: dma address of transfer ring.
+ * @free_trbs: number of free TRBs in transfer ring.
+ * @pcs: producer cycle state.
+ * @ccs: consumer cycle state.
+ * @enqueue: enqueue index in transfer ring.
+ * @dequeue: dequeue index in transfer ring.
+ */
+struct cdns2_ring {
+ struct cdns2_trb *trbs;
+ dma_addr_t dma;
+ int free_trbs;
+ u8 pcs;
+ u8 ccs;
+ int enqueue;
+ int dequeue;
+};
+
+/**
+ * struct cdns2_endpoint - extended device side representation of USB endpoint.
+ * @endpoint: usb endpoint.
+ * @pending_list: list of requests queuing on transfer ring.
+ * @deferred_list: list of requests waiting for queuing on transfer ring.
+ * @pdev: device associated with endpoint.
+ * @name: a human readable name e.g. ep1out.
+ * @ring: transfer ring associated with endpoint.
+ * @ep_state: state of endpoint.
+ * @idx: index of endpoint in pdev->eps table.
+ * @dir: endpoint direction.
+ * @num: endpoint number (1 - 15).
+ * @type: set to bmAttributes & USB_ENDPOINT_XFERTYPE_MASK.
+ * @interval: interval between packets used for ISOC and Interrupt endpoint.
+ * @buffering: on-chip buffers assigned to endpoint.
+ * @trb_burst_size: number of burst used in TRB.
+ * @skip: Sometimes the controller cannot process isochronous endpoint ring
+ * quickly enough and it will miss some isoc tds on the ring and
+ * generate ISO transmition error.
+ * Driver sets skip flag when receive a ISO transmition error and
+ * process the missed TDs on the endpoint ring.
+ * @wa1_set: use WA1.
+ * @wa1_trb: TRB assigned to WA1.
+ * @wa1_trb_index: TRB index for WA1.
+ * @wa1_cycle_bit: correct cycle bit for WA1.
+ */
+struct cdns2_endpoint {
+ struct usb_ep endpoint;
+ struct list_head pending_list;
+ struct list_head deferred_list;
+
+ struct cdns2_device *pdev;
+ char name[20];
+
+ struct cdns2_ring ring;
+
+#define EP_ENABLED BIT(0)
+#define EP_STALLED BIT(1)
+#define EP_STALL_PENDING BIT(2)
+#define EP_WEDGE BIT(3)
+#define EP_CLAIMED BIT(4)
+#define EP_RING_FULL BIT(5)
+#define EP_DEFERRED_DRDY BIT(6)
+
+ u32 ep_state;
+
+ u8 idx;
+ u8 dir;
+ u8 num;
+ u8 type;
+ int interval;
+ u8 buffering;
+ u8 trb_burst_size;
+ bool skip;
+
+ unsigned int wa1_set:1;
+ struct cdns2_trb *wa1_trb;
+ unsigned int wa1_trb_index;
+ unsigned int wa1_cycle_bit:1;
+};
+
+/**
+ * struct cdns2_request - extended device side representation of usb_request
+ * object.
+ * @request: generic usb_request object describing single I/O request.
+ * @pep: extended representation of usb_ep object.
+ * @trb: the first TRB association with this request.
+ * @start_trb: number of the first TRB in transfer ring.
+ * @end_trb: number of the last TRB in transfer ring.
+ * @list: used for queuing request in lists.
+ * @finished_trb: number of trb has already finished per request.
+ * @num_of_trb: how many trbs are associated with request.
+ */
+struct cdns2_request {
+ struct usb_request request;
+ struct cdns2_endpoint *pep;
+ struct cdns2_trb *trb;
+ int start_trb;
+ int end_trb;
+ struct list_head list;
+ int finished_trb;
+ int num_of_trb;
+};
+
+#define to_cdns2_request(r) (container_of(r, struct cdns2_request, request))
+
+/* Stages used during enumeration process.*/
+#define CDNS2_SETUP_STAGE 0x0
+#define CDNS2_DATA_STAGE 0x1
+#define CDNS2_STATUS_STAGE 0x2
+
+/**
+ * struct cdns2_device - represent USB device.
+ * @dev: pointer to device structure associated whit this controller.
+ * @gadget: device side representation of the peripheral controller.
+ * @gadget_driver: pointer to the gadget driver.
+ * @lock: for synchronizing.
+ * @irq: interrupt line number.
+ * @regs: base address for registers
+ * @usb_regs: base address for common USB registers.
+ * @ep0_regs: base address for endpoint 0 related registers.
+ * @epx_regs: base address for all none control endpoint registers.
+ * @interrupt_regs: base address for interrupt handling related registers.
+ * @adma_regs: base address for ADMA registers.
+ * @eps_dma_pool: endpoint Transfer Ring pool.
+ * @setup: used while processing usb control requests.
+ * @ep0_preq: private request used while handling EP0.
+ * @ep0_stage: ep0 stage during enumeration process.
+ * @zlp_buf: zlp buffer.
+ * @dev_address: device address assigned by host.
+ * @eps: array of objects describing endpoints.
+ * @selected_ep: actually selected endpoint. It's used only to improve
+ * performance by limiting access to dma_ep_sel register.
+ * @is_selfpowered: device is self powered.
+ * @may_wakeup: allows device to remote wakeup the host.
+ * @status_completion_no_call: indicate that driver is waiting for status
+ * stage completion. It's used in deferred SET_CONFIGURATION request.
+ * @in_lpm: indicate the controller is in low power mode.
+ * @pending_status_wq: workqueue handling status stage for deferred requests.
+ * @pending_status_request: request for which status stage was deferred.
+ * @eps_supported: endpoints supported by controller in form:
+ * bit: 0 - ep0, 1 - epOut1, 2 - epIn1, 3 - epOut2 ...
+ * @burst_opt: array with the best burst size value for different TRB size.
+ * @onchip_tx_buf: size of transmit on-chip buffer in KB.
+ * @onchip_rx_buf: size of receive on-chip buffer in KB.
+ */
+struct cdns2_device {
+ struct device *dev;
+ struct usb_gadget gadget;
+ struct usb_gadget_driver *gadget_driver;
+
+ /* generic spin-lock for drivers */
+ spinlock_t lock;
+ int irq;
+ void __iomem *regs;
+ struct cdns2_usb_regs __iomem *usb_regs;
+ struct cdns2_ep0_regs __iomem *ep0_regs;
+ struct cdns2_epx_regs __iomem *epx_regs;
+ struct cdns2_interrupt_regs __iomem *interrupt_regs;
+ struct cdns2_adma_regs __iomem *adma_regs;
+ struct dma_pool *eps_dma_pool;
+ struct usb_ctrlrequest setup;
+ struct cdns2_request ep0_preq;
+ u8 ep0_stage;
+ void *zlp_buf;
+ u8 dev_address;
+ struct cdns2_endpoint eps[CDNS2_ENDPOINTS_NUM];
+ u32 selected_ep;
+ bool is_selfpowered;
+ bool may_wakeup;
+ bool status_completion_no_call;
+ bool in_lpm;
+ struct work_struct pending_status_wq;
+ struct usb_request *pending_status_request;
+ u32 eps_supported;
+ u8 burst_opt[MAX_ISO_SIZE + 1];
+
+ /*in KB */
+ u16 onchip_tx_buf;
+ u16 onchip_rx_buf;
+};
+
+#define CDNS2_IF_EP_EXIST(pdev, ep_num, dir) \
+ ((pdev)->eps_supported & \
+ (BIT(ep_num) << ((dir) ? 0 : 16)))
+
+dma_addr_t cdns2_trb_virt_to_dma(struct cdns2_endpoint *pep,
+ struct cdns2_trb *trb);
+void cdns2_pending_setup_status_handler(struct work_struct *work);
+void cdns2_select_ep(struct cdns2_device *pdev, u32 ep);
+struct cdns2_request *cdns2_next_preq(struct list_head *list);
+struct usb_request *cdns2_gadget_ep_alloc_request(struct usb_ep *ep,
+ gfp_t gfp_flags);
+void cdns2_gadget_ep_free_request(struct usb_ep *ep,
+ struct usb_request *request);
+int cdns2_gadget_ep_dequeue(struct usb_ep *ep, struct usb_request *request);
+void cdns2_gadget_giveback(struct cdns2_endpoint *pep,
+ struct cdns2_request *priv_req,
+ int status);
+void cdns2_init_ep0(struct cdns2_device *pdev, struct cdns2_endpoint *pep);
+void cdns2_ep0_config(struct cdns2_device *pdev);
+void cdns2_handle_ep0_interrupt(struct cdns2_device *pdev, int dir);
+void cdns2_handle_setup_packet(struct cdns2_device *pdev);
+int cdns2_gadget_resume(struct cdns2_device *pdev, bool hibernated);
+int cdns2_gadget_suspend(struct cdns2_device *pdev);
+void cdns2_gadget_remove(struct cdns2_device *pdev);
+int cdns2_gadget_init(struct cdns2_device *pdev);
+void set_reg_bit_8(void __iomem *ptr, u8 mask);
+int cdns2_halt_endpoint(struct cdns2_device *pdev, struct cdns2_endpoint *pep,
+ int value);
+
+#endif /* __LINUX_CDNS2_GADGET */
diff --git a/drivers/usb/gadget/udc/cdns2/cdns2-pci.c b/drivers/usb/gadget/udc/cdns2/cdns2-pci.c
new file mode 100644
index 0000000000..1691541c94
--- /dev/null
+++ b/drivers/usb/gadget/udc/cdns2/cdns2-pci.c
@@ -0,0 +1,138 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Cadence USBHS-DEV controller - PCI Glue driver.
+ *
+ * Copyright (C) 2023 Cadence.
+ *
+ * Author: Pawel Laszczak <pawell@cadence.com>
+ *
+ */
+
+#include <linux/pm_runtime.h>
+#include <linux/slab.h>
+#include <linux/pci.h>
+
+#include "cdns2-gadget.h"
+
+#define PCI_DRIVER_NAME "cdns-pci-usbhs"
+#define CDNS_VENDOR_ID 0x17cd
+#define CDNS_DEVICE_ID 0x0120
+#define PCI_BAR_DEV 0
+#define PCI_DEV_FN_DEVICE 0
+
+static int cdns2_pci_probe(struct pci_dev *pdev,
+ const struct pci_device_id *id)
+{
+ resource_size_t rsrc_start, rsrc_len;
+ struct device *dev = &pdev->dev;
+ struct cdns2_device *priv_dev;
+ struct resource *res;
+ int ret;
+
+ /* For GADGET PCI (devfn) function number is 0. */
+ if (!id || pdev->devfn != PCI_DEV_FN_DEVICE ||
+ pdev->class != PCI_CLASS_SERIAL_USB_DEVICE)
+ return -EINVAL;
+
+ ret = pcim_enable_device(pdev);
+ if (ret) {
+ dev_err(&pdev->dev, "Enabling PCI device has failed %d\n", ret);
+ return ret;
+ }
+
+ pci_set_master(pdev);
+
+ priv_dev = devm_kzalloc(&pdev->dev, sizeof(*priv_dev), GFP_KERNEL);
+ if (!priv_dev)
+ return -ENOMEM;
+
+ dev_dbg(dev, "Initialize resources\n");
+ rsrc_start = pci_resource_start(pdev, PCI_BAR_DEV);
+ rsrc_len = pci_resource_len(pdev, PCI_BAR_DEV);
+
+ res = devm_request_mem_region(dev, rsrc_start, rsrc_len, "dev");
+ if (!res) {
+ dev_dbg(dev, "controller already in use\n");
+ return -EBUSY;
+ }
+
+ priv_dev->regs = devm_ioremap(dev, rsrc_start, rsrc_len);
+ if (!priv_dev->regs) {
+ dev_dbg(dev, "error mapping memory\n");
+ return -EFAULT;
+ }
+
+ priv_dev->irq = pdev->irq;
+ dev_dbg(dev, "USBSS-DEV physical base addr: %pa\n",
+ &rsrc_start);
+
+ priv_dev->dev = dev;
+
+ priv_dev->eps_supported = 0x000f000f;
+ priv_dev->onchip_tx_buf = 16;
+ priv_dev->onchip_rx_buf = 16;
+
+ ret = cdns2_gadget_init(priv_dev);
+ if (ret)
+ return ret;
+
+ pci_set_drvdata(pdev, priv_dev);
+
+ device_wakeup_enable(&pdev->dev);
+ if (pci_dev_run_wake(pdev))
+ pm_runtime_put_noidle(&pdev->dev);
+
+ return 0;
+}
+
+static void cdns2_pci_remove(struct pci_dev *pdev)
+{
+ struct cdns2_device *priv_dev = pci_get_drvdata(pdev);
+
+ if (pci_dev_run_wake(pdev))
+ pm_runtime_get_noresume(&pdev->dev);
+
+ cdns2_gadget_remove(priv_dev);
+}
+
+static int cdns2_pci_suspend(struct device *dev)
+{
+ struct cdns2_device *priv_dev = dev_get_drvdata(dev);
+
+ return cdns2_gadget_suspend(priv_dev);
+}
+
+static int cdns2_pci_resume(struct device *dev)
+{
+ struct cdns2_device *priv_dev = dev_get_drvdata(dev);
+
+ return cdns2_gadget_resume(priv_dev, 1);
+}
+
+static const struct dev_pm_ops cdns2_pci_pm_ops = {
+ SYSTEM_SLEEP_PM_OPS(cdns2_pci_suspend, cdns2_pci_resume)
+};
+
+static const struct pci_device_id cdns2_pci_ids[] = {
+ { PCI_VENDOR_ID_CDNS, CDNS_DEVICE_ID, PCI_ANY_ID, PCI_ANY_ID,
+ PCI_CLASS_SERIAL_USB_DEVICE, PCI_ANY_ID },
+ { 0, }
+};
+
+static struct pci_driver cdns2_pci_driver = {
+ .name = "cdns2-pci",
+ .id_table = &cdns2_pci_ids[0],
+ .probe = cdns2_pci_probe,
+ .remove = cdns2_pci_remove,
+ .driver = {
+ .pm = pm_ptr(&cdns2_pci_pm_ops),
+ }
+};
+
+module_pci_driver(cdns2_pci_driver);
+MODULE_DEVICE_TABLE(pci, cdns2_pci_ids);
+
+MODULE_ALIAS("pci:cdns2");
+MODULE_AUTHOR("Pawel Laszczak <pawell@cadence.com>");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Cadence CDNS2 PCI driver");
diff --git a/drivers/usb/gadget/udc/cdns2/cdns2-trace.c b/drivers/usb/gadget/udc/cdns2/cdns2-trace.c
new file mode 100644
index 0000000000..de6b8cc3d0
--- /dev/null
+++ b/drivers/usb/gadget/udc/cdns2/cdns2-trace.c
@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * USBHS device controller driver Trace Support
+ *
+ * Copyright (C) 2023 Cadence.
+ *
+ * Author: Pawel Laszczak <pawell@cadence.com>
+ */
+
+#define CREATE_TRACE_POINTS
+#include "cdns2-trace.h"
diff --git a/drivers/usb/gadget/udc/cdns2/cdns2-trace.h b/drivers/usb/gadget/udc/cdns2/cdns2-trace.h
new file mode 100644
index 0000000000..61f241634e
--- /dev/null
+++ b/drivers/usb/gadget/udc/cdns2/cdns2-trace.h
@@ -0,0 +1,605 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * USBHS-DEV device controller driver.
+ * Trace support header file.
+ *
+ * Copyright (C) 2023 Cadence.
+ *
+ * Author: Pawel Laszczak <pawell@cadence.com>
+ */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM cdns2-dev
+
+/*
+ * The TRACE_SYSTEM_VAR defaults to TRACE_SYSTEM, but must be a
+ * legitimate C variable. It is not exported to user space.
+ */
+#undef TRACE_SYSTEM_VAR
+#define TRACE_SYSTEM_VAR cdns2_dev
+
+#if !defined(__LINUX_CDNS2_TRACE) || defined(TRACE_HEADER_MULTI_READ)
+#define __LINUX_CDNS2_TRACE
+
+#include <linux/types.h>
+#include <linux/tracepoint.h>
+#include <asm/byteorder.h>
+#include <linux/usb/ch9.h>
+#include "cdns2-gadget.h"
+#include "cdns2-debug.h"
+
+#define CDNS2_MSG_MAX 500
+
+DECLARE_EVENT_CLASS(cdns2_log_enable_disable,
+ TP_PROTO(int set),
+ TP_ARGS(set),
+ TP_STRUCT__entry(
+ __field(int, set)
+ ),
+ TP_fast_assign(
+ __entry->set = set;
+ ),
+ TP_printk("%s", __entry->set ? "enabled" : "disabled")
+);
+
+DEFINE_EVENT(cdns2_log_enable_disable, cdns2_pullup,
+ TP_PROTO(int set),
+ TP_ARGS(set)
+);
+
+DEFINE_EVENT(cdns2_log_enable_disable, cdns2_lpm,
+ TP_PROTO(int set),
+ TP_ARGS(set)
+);
+
+DEFINE_EVENT(cdns2_log_enable_disable, cdns2_may_wakeup,
+ TP_PROTO(int set),
+ TP_ARGS(set)
+);
+
+DECLARE_EVENT_CLASS(cdns2_log_simple,
+ TP_PROTO(char *msg),
+ TP_ARGS(msg),
+ TP_STRUCT__entry(
+ __string(text, msg)
+ ),
+ TP_fast_assign(
+ __assign_str(text, msg);
+ ),
+ TP_printk("%s", __get_str(text))
+);
+
+DEFINE_EVENT(cdns2_log_simple, cdns2_no_room_on_ring,
+ TP_PROTO(char *msg),
+ TP_ARGS(msg)
+);
+
+DEFINE_EVENT(cdns2_log_simple, cdns2_ep0_status_stage,
+ TP_PROTO(char *msg),
+ TP_ARGS(msg)
+);
+
+DEFINE_EVENT(cdns2_log_simple, cdns2_ep0_set_config,
+ TP_PROTO(char *msg),
+ TP_ARGS(msg)
+);
+
+DEFINE_EVENT(cdns2_log_simple, cdns2_ep0_setup,
+ TP_PROTO(char *msg),
+ TP_ARGS(msg)
+);
+
+DEFINE_EVENT(cdns2_log_simple, cdns2_device_state,
+ TP_PROTO(char *msg),
+ TP_ARGS(msg)
+);
+
+TRACE_EVENT(cdns2_ep_halt,
+ TP_PROTO(struct cdns2_endpoint *ep_priv, u8 halt, u8 flush),
+ TP_ARGS(ep_priv, halt, flush),
+ TP_STRUCT__entry(
+ __string(name, ep_priv->name)
+ __field(u8, halt)
+ __field(u8, flush)
+ ),
+ TP_fast_assign(
+ __assign_str(name, ep_priv->name);
+ __entry->halt = halt;
+ __entry->flush = flush;
+ ),
+ TP_printk("Halt %s for %s: %s", __entry->flush ? " and flush" : "",
+ __get_str(name), __entry->halt ? "set" : "cleared")
+);
+
+TRACE_EVENT(cdns2_wa1,
+ TP_PROTO(struct cdns2_endpoint *ep_priv, char *msg),
+ TP_ARGS(ep_priv, msg),
+ TP_STRUCT__entry(
+ __string(ep_name, ep_priv->name)
+ __string(msg, msg)
+ ),
+ TP_fast_assign(
+ __assign_str(ep_name, ep_priv->name);
+ __assign_str(msg, msg);
+ ),
+ TP_printk("WA1: %s %s", __get_str(ep_name), __get_str(msg))
+);
+
+DECLARE_EVENT_CLASS(cdns2_log_doorbell,
+ TP_PROTO(struct cdns2_endpoint *pep, u32 ep_trbaddr),
+ TP_ARGS(pep, ep_trbaddr),
+ TP_STRUCT__entry(
+ __string(name, pep->num ? pep->name :
+ (pep->dir ? "ep0in" : "ep0out"))
+ __field(u32, ep_trbaddr)
+ ),
+ TP_fast_assign(
+ __assign_str(name, pep->name);
+ __entry->ep_trbaddr = ep_trbaddr;
+ ),
+ TP_printk("%s, ep_trbaddr %08x", __get_str(name),
+ __entry->ep_trbaddr)
+);
+
+DEFINE_EVENT(cdns2_log_doorbell, cdns2_doorbell_ep0,
+ TP_PROTO(struct cdns2_endpoint *pep, u32 ep_trbaddr),
+ TP_ARGS(pep, ep_trbaddr)
+);
+
+DEFINE_EVENT(cdns2_log_doorbell, cdns2_doorbell_epx,
+ TP_PROTO(struct cdns2_endpoint *pep, u32 ep_trbaddr),
+ TP_ARGS(pep, ep_trbaddr)
+);
+
+DECLARE_EVENT_CLASS(cdns2_log_usb_irq,
+ TP_PROTO(u32 usb_irq, u32 ext_irq),
+ TP_ARGS(usb_irq, ext_irq),
+ TP_STRUCT__entry(
+ __field(u32, usb_irq)
+ __field(u32, ext_irq)
+ ),
+ TP_fast_assign(
+ __entry->usb_irq = usb_irq;
+ __entry->ext_irq = ext_irq;
+ ),
+ TP_printk("%s", cdns2_decode_usb_irq(__get_buf(CDNS2_MSG_MAX),
+ CDNS2_MSG_MAX,
+ __entry->usb_irq,
+ __entry->ext_irq))
+);
+
+DEFINE_EVENT(cdns2_log_usb_irq, cdns2_usb_irq,
+ TP_PROTO(u32 usb_irq, u32 ext_irq),
+ TP_ARGS(usb_irq, ext_irq)
+);
+
+TRACE_EVENT(cdns2_dma_ep_ists,
+ TP_PROTO(u32 dma_ep_ists),
+ TP_ARGS(dma_ep_ists),
+ TP_STRUCT__entry(
+ __field(u32, dma_ep_ists)
+ ),
+ TP_fast_assign(
+ __entry->dma_ep_ists = dma_ep_ists;
+ ),
+ TP_printk("OUT: 0x%04x, IN: 0x%04x", (u16)__entry->dma_ep_ists,
+ __entry->dma_ep_ists >> 16)
+);
+
+DECLARE_EVENT_CLASS(cdns2_log_epx_irq,
+ TP_PROTO(struct cdns2_device *pdev, struct cdns2_endpoint *pep),
+ TP_ARGS(pdev, pep),
+ TP_STRUCT__entry(
+ __string(ep_name, pep->name)
+ __field(u32, ep_sts)
+ __field(u32, ep_ists)
+ __field(u32, ep_traddr)
+ ),
+ TP_fast_assign(
+ __assign_str(ep_name, pep->name);
+ __entry->ep_sts = readl(&pdev->adma_regs->ep_sts);
+ __entry->ep_ists = readl(&pdev->adma_regs->ep_ists);
+ __entry->ep_traddr = readl(&pdev->adma_regs->ep_traddr);
+ ),
+ TP_printk("%s, ep_traddr: %08x",
+ cdns2_decode_epx_irq(__get_buf(CDNS2_MSG_MAX), CDNS2_MSG_MAX,
+ __get_str(ep_name),
+ __entry->ep_ists, __entry->ep_sts),
+ __entry->ep_traddr)
+);
+
+DEFINE_EVENT(cdns2_log_epx_irq, cdns2_epx_irq,
+ TP_PROTO(struct cdns2_device *pdev, struct cdns2_endpoint *pep),
+ TP_ARGS(pdev, pep)
+);
+
+DECLARE_EVENT_CLASS(cdns2_log_ep0_irq,
+ TP_PROTO(struct cdns2_device *pdev),
+ TP_ARGS(pdev),
+ TP_STRUCT__entry(
+ __field(int, ep_dir)
+ __field(u32, ep_ists)
+ __field(u32, ep_sts)
+ ),
+ TP_fast_assign(
+ __entry->ep_dir = pdev->selected_ep;
+ __entry->ep_ists = readl(&pdev->adma_regs->ep_ists);
+ __entry->ep_sts = readl(&pdev->adma_regs->ep_sts);
+ ),
+ TP_printk("%s", cdns2_decode_ep0_irq(__get_buf(CDNS2_MSG_MAX),
+ CDNS2_MSG_MAX,
+ __entry->ep_ists, __entry->ep_sts,
+ __entry->ep_dir))
+);
+
+DEFINE_EVENT(cdns2_log_ep0_irq, cdns2_ep0_irq,
+ TP_PROTO(struct cdns2_device *pdev),
+ TP_ARGS(pdev)
+);
+
+DECLARE_EVENT_CLASS(cdns2_log_ctrl,
+ TP_PROTO(struct usb_ctrlrequest *ctrl),
+ TP_ARGS(ctrl),
+ TP_STRUCT__entry(
+ __field(u8, bRequestType)
+ __field(u8, bRequest)
+ __field(u16, wValue)
+ __field(u16, wIndex)
+ __field(u16, wLength)
+ ),
+ TP_fast_assign(
+ __entry->bRequestType = ctrl->bRequestType;
+ __entry->bRequest = ctrl->bRequest;
+ __entry->wValue = le16_to_cpu(ctrl->wValue);
+ __entry->wIndex = le16_to_cpu(ctrl->wIndex);
+ __entry->wLength = le16_to_cpu(ctrl->wLength);
+ ),
+ TP_printk("%s", usb_decode_ctrl(__get_buf(CDNS2_MSG_MAX), CDNS2_MSG_MAX,
+ __entry->bRequestType,
+ __entry->bRequest, __entry->wValue,
+ __entry->wIndex, __entry->wLength)
+ )
+);
+
+DEFINE_EVENT(cdns2_log_ctrl, cdns2_ctrl_req,
+ TP_PROTO(struct usb_ctrlrequest *ctrl),
+ TP_ARGS(ctrl)
+);
+
+DECLARE_EVENT_CLASS(cdns2_log_request,
+ TP_PROTO(struct cdns2_request *preq),
+ TP_ARGS(preq),
+ TP_STRUCT__entry(
+ __string(name, preq->pep->name)
+ __field(struct usb_request *, request)
+ __field(struct cdns2_request *, preq)
+ __field(void *, buf)
+ __field(unsigned int, actual)
+ __field(unsigned int, length)
+ __field(int, status)
+ __field(dma_addr_t, dma)
+ __field(int, zero)
+ __field(int, short_not_ok)
+ __field(int, no_interrupt)
+ __field(struct scatterlist*, sg)
+ __field(unsigned int, num_sgs)
+ __field(unsigned int, num_mapped_sgs)
+ __field(int, start_trb)
+ __field(int, end_trb)
+ ),
+ TP_fast_assign(
+ __assign_str(name, preq->pep->name);
+ __entry->request = &preq->request;
+ __entry->preq = preq;
+ __entry->buf = preq->request.buf;
+ __entry->actual = preq->request.actual;
+ __entry->length = preq->request.length;
+ __entry->status = preq->request.status;
+ __entry->dma = preq->request.dma;
+ __entry->zero = preq->request.zero;
+ __entry->short_not_ok = preq->request.short_not_ok;
+ __entry->no_interrupt = preq->request.no_interrupt;
+ __entry->sg = preq->request.sg;
+ __entry->num_sgs = preq->request.num_sgs;
+ __entry->num_mapped_sgs = preq->request.num_mapped_sgs;
+ __entry->start_trb = preq->start_trb;
+ __entry->end_trb = preq->end_trb;
+ ),
+ TP_printk("%s: req: %p, preq: %p, req buf: %p, length: %u/%u, status: %d,"
+ "buf dma: (%pad), %s%s%s, sg: %p, num_sgs: %d, num_m_sgs: %d,"
+ "trb: [start: %d, end: %d]",
+ __get_str(name), __entry->request, __entry->preq,
+ __entry->buf, __entry->actual, __entry->length,
+ __entry->status, &__entry->dma,
+ __entry->zero ? "Z" : "z",
+ __entry->short_not_ok ? "S" : "s",
+ __entry->no_interrupt ? "I" : "i",
+ __entry->sg, __entry->num_sgs, __entry->num_mapped_sgs,
+ __entry->start_trb,
+ __entry->end_trb
+ )
+);
+
+DEFINE_EVENT(cdns2_log_request, cdns2_request_enqueue,
+ TP_PROTO(struct cdns2_request *preq),
+ TP_ARGS(preq)
+);
+
+DEFINE_EVENT(cdns2_log_request, cdns2_request_enqueue_error,
+ TP_PROTO(struct cdns2_request *preq),
+ TP_ARGS(preq)
+);
+
+DEFINE_EVENT(cdns2_log_request, cdns2_alloc_request,
+ TP_PROTO(struct cdns2_request *preq),
+ TP_ARGS(preq)
+);
+
+DEFINE_EVENT(cdns2_log_request, cdns2_free_request,
+ TP_PROTO(struct cdns2_request *preq),
+ TP_ARGS(preq)
+);
+
+DEFINE_EVENT(cdns2_log_request, cdns2_ep_queue,
+ TP_PROTO(struct cdns2_request *preq),
+ TP_ARGS(preq)
+);
+
+DEFINE_EVENT(cdns2_log_request, cdns2_request_dequeue,
+ TP_PROTO(struct cdns2_request *preq),
+ TP_ARGS(preq)
+);
+
+DEFINE_EVENT(cdns2_log_request, cdns2_request_giveback,
+ TP_PROTO(struct cdns2_request *preq),
+ TP_ARGS(preq)
+);
+
+TRACE_EVENT(cdns2_ep0_enqueue,
+ TP_PROTO(struct cdns2_device *dev_priv, struct usb_request *request),
+ TP_ARGS(dev_priv, request),
+ TP_STRUCT__entry(
+ __field(int, dir)
+ __field(int, length)
+ ),
+ TP_fast_assign(
+ __entry->dir = dev_priv->eps[0].dir;
+ __entry->length = request->length;
+ ),
+ TP_printk("Queue to ep0%s length: %u", __entry->dir ? "in" : "out",
+ __entry->length)
+);
+
+DECLARE_EVENT_CLASS(cdns2_log_map_request,
+ TP_PROTO(struct cdns2_request *priv_req),
+ TP_ARGS(priv_req),
+ TP_STRUCT__entry(
+ __string(name, priv_req->pep->name)
+ __field(struct usb_request *, req)
+ __field(void *, buf)
+ __field(dma_addr_t, dma)
+ ),
+ TP_fast_assign(
+ __assign_str(name, priv_req->pep->name);
+ __entry->req = &priv_req->request;
+ __entry->buf = priv_req->request.buf;
+ __entry->dma = priv_req->request.dma;
+ ),
+ TP_printk("%s: req: %p, req buf %p, dma %p",
+ __get_str(name), __entry->req, __entry->buf, &__entry->dma
+ )
+);
+
+DEFINE_EVENT(cdns2_log_map_request, cdns2_map_request,
+ TP_PROTO(struct cdns2_request *req),
+ TP_ARGS(req)
+);
+DEFINE_EVENT(cdns2_log_map_request, cdns2_mapped_request,
+ TP_PROTO(struct cdns2_request *req),
+ TP_ARGS(req)
+);
+
+DECLARE_EVENT_CLASS(cdns2_log_trb,
+ TP_PROTO(struct cdns2_endpoint *pep, struct cdns2_trb *trb),
+ TP_ARGS(pep, trb),
+ TP_STRUCT__entry(
+ __string(name, pep->name)
+ __field(struct cdns2_trb *, trb)
+ __field(u32, buffer)
+ __field(u32, length)
+ __field(u32, control)
+ __field(u32, type)
+ ),
+ TP_fast_assign(
+ __assign_str(name, pep->name);
+ __entry->trb = trb;
+ __entry->buffer = le32_to_cpu(trb->buffer);
+ __entry->length = le32_to_cpu(trb->length);
+ __entry->control = le32_to_cpu(trb->control);
+ __entry->type = usb_endpoint_type(pep->endpoint.desc);
+ ),
+ TP_printk("%s: trb V: %p, dma buf: P: 0x%08x, %s",
+ __get_str(name), __entry->trb, __entry->buffer,
+ cdns2_decode_trb(__get_buf(CDNS2_MSG_MAX), CDNS2_MSG_MAX,
+ __entry->control, __entry->length,
+ __entry->buffer))
+);
+
+DEFINE_EVENT(cdns2_log_trb, cdns2_queue_trb,
+ TP_PROTO(struct cdns2_endpoint *pep, struct cdns2_trb *trb),
+ TP_ARGS(pep, trb)
+);
+
+DEFINE_EVENT(cdns2_log_trb, cdns2_complete_trb,
+ TP_PROTO(struct cdns2_endpoint *pep, struct cdns2_trb *trb),
+ TP_ARGS(pep, trb)
+);
+
+DECLARE_EVENT_CLASS(cdns2_log_ring,
+ TP_PROTO(struct cdns2_endpoint *pep),
+ TP_ARGS(pep),
+ TP_STRUCT__entry(
+ __dynamic_array(u8, tr_seg, TR_SEG_SIZE)
+ __dynamic_array(u8, pep, sizeof(struct cdns2_endpoint))
+ __dynamic_array(char, buffer,
+ (TRBS_PER_SEGMENT * 65) + CDNS2_MSG_MAX)
+ ),
+ TP_fast_assign(
+ memcpy(__get_dynamic_array(pep), pep,
+ sizeof(struct cdns2_endpoint));
+ memcpy(__get_dynamic_array(tr_seg), pep->ring.trbs,
+ TR_SEG_SIZE);
+ ),
+
+ TP_printk("%s",
+ cdns2_raw_ring((struct cdns2_endpoint *)__get_str(pep),
+ (struct cdns2_trb *)__get_str(tr_seg),
+ __get_str(buffer),
+ (TRBS_PER_SEGMENT * 65) + CDNS2_MSG_MAX))
+);
+
+DEFINE_EVENT(cdns2_log_ring, cdns2_ring,
+ TP_PROTO(struct cdns2_endpoint *pep),
+ TP_ARGS(pep)
+);
+
+DECLARE_EVENT_CLASS(cdns2_log_ep,
+ TP_PROTO(struct cdns2_endpoint *pep),
+ TP_ARGS(pep),
+ TP_STRUCT__entry(
+ __string(name, pep->name)
+ __field(unsigned int, maxpacket)
+ __field(unsigned int, maxpacket_limit)
+ __field(unsigned int, flags)
+ __field(unsigned int, dir)
+ __field(u8, enqueue)
+ __field(u8, dequeue)
+ ),
+ TP_fast_assign(
+ __assign_str(name, pep->name);
+ __entry->maxpacket = pep->endpoint.maxpacket;
+ __entry->maxpacket_limit = pep->endpoint.maxpacket_limit;
+ __entry->flags = pep->ep_state;
+ __entry->dir = pep->dir;
+ __entry->enqueue = pep->ring.enqueue;
+ __entry->dequeue = pep->ring.dequeue;
+ ),
+ TP_printk("%s: mps: %d/%d, enq idx: %d, deq idx: %d, "
+ "flags: %s%s%s%s, dir: %s",
+ __get_str(name), __entry->maxpacket,
+ __entry->maxpacket_limit, __entry->enqueue,
+ __entry->dequeue,
+ __entry->flags & EP_ENABLED ? "EN | " : "",
+ __entry->flags & EP_STALLED ? "STALLED | " : "",
+ __entry->flags & EP_WEDGE ? "WEDGE | " : "",
+ __entry->flags & EP_RING_FULL ? "RING FULL |" : "",
+ __entry->dir ? "IN" : "OUT"
+ )
+);
+
+DEFINE_EVENT(cdns2_log_ep, cdns2_gadget_ep_enable,
+ TP_PROTO(struct cdns2_endpoint *pep),
+ TP_ARGS(pep)
+);
+
+DEFINE_EVENT(cdns2_log_ep, cdns2_gadget_ep_disable,
+ TP_PROTO(struct cdns2_endpoint *pep),
+ TP_ARGS(pep)
+);
+
+DEFINE_EVENT(cdns2_log_ep, cdns2_iso_out_ep_disable,
+ TP_PROTO(struct cdns2_endpoint *pep),
+ TP_ARGS(pep)
+);
+
+DEFINE_EVENT(cdns2_log_ep, cdns2_ep_busy_try_halt_again,
+ TP_PROTO(struct cdns2_endpoint *pep),
+ TP_ARGS(pep)
+);
+
+DECLARE_EVENT_CLASS(cdns2_log_request_handled,
+ TP_PROTO(struct cdns2_request *priv_req, int current_index,
+ int handled),
+ TP_ARGS(priv_req, current_index, handled),
+ TP_STRUCT__entry(
+ __field(struct cdns2_request *, priv_req)
+ __field(unsigned int, dma_position)
+ __field(unsigned int, handled)
+ __field(unsigned int, dequeue_idx)
+ __field(unsigned int, enqueue_idx)
+ __field(unsigned int, start_trb)
+ __field(unsigned int, end_trb)
+ ),
+ TP_fast_assign(
+ __entry->priv_req = priv_req;
+ __entry->dma_position = current_index;
+ __entry->handled = handled;
+ __entry->dequeue_idx = priv_req->pep->ring.dequeue;
+ __entry->enqueue_idx = priv_req->pep->ring.enqueue;
+ __entry->start_trb = priv_req->start_trb;
+ __entry->end_trb = priv_req->end_trb;
+ ),
+ TP_printk("Req: %p %s, DMA pos: %d, ep deq: %d, ep enq: %d,"
+ " start trb: %d, end trb: %d",
+ __entry->priv_req,
+ __entry->handled ? "handled" : "not handled",
+ __entry->dma_position, __entry->dequeue_idx,
+ __entry->enqueue_idx, __entry->start_trb,
+ __entry->end_trb
+ )
+);
+
+DEFINE_EVENT(cdns2_log_request_handled, cdns2_request_handled,
+ TP_PROTO(struct cdns2_request *priv_req, int current_index,
+ int handled),
+ TP_ARGS(priv_req, current_index, handled)
+);
+
+DECLARE_EVENT_CLASS(cdns2_log_epx_reg_config,
+ TP_PROTO(struct cdns2_device *pdev, struct cdns2_endpoint *pep),
+ TP_ARGS(pdev, pep),
+ TP_STRUCT__entry(
+ __string(ep_name, pep->name)
+ __field(u8, burst_size)
+ __field(__le16, maxpack_reg)
+ __field(__u8, con_reg)
+ __field(u32, ep_sel_reg)
+ __field(u32, ep_sts_en_reg)
+ __field(u32, ep_cfg_reg)
+ ),
+ TP_fast_assign(
+ __assign_str(ep_name, pep->name);
+ __entry->burst_size = pep->trb_burst_size;
+ __entry->maxpack_reg = pep->dir ? readw(&pdev->epx_regs->txmaxpack[pep->num - 1]) :
+ readw(&pdev->epx_regs->rxmaxpack[pep->num - 1]);
+ __entry->con_reg = pep->dir ? readb(&pdev->epx_regs->ep[pep->num - 1].txcon) :
+ readb(&pdev->epx_regs->ep[pep->num - 1].rxcon);
+ __entry->ep_sel_reg = readl(&pdev->adma_regs->ep_sel);
+ __entry->ep_sts_en_reg = readl(&pdev->adma_regs->ep_sts_en);
+ __entry->ep_cfg_reg = readl(&pdev->adma_regs->ep_cfg);
+ ),
+
+ TP_printk("%s, maxpack: %d, con: %02x, dma_ep_sel: %08x, dma_ep_sts_en: %08x"
+ " dma_ep_cfg %08x",
+ __get_str(ep_name), __entry->maxpack_reg, __entry->con_reg,
+ __entry->ep_sel_reg, __entry->ep_sts_en_reg,
+ __entry->ep_cfg_reg
+ )
+);
+
+DEFINE_EVENT(cdns2_log_epx_reg_config, cdns2_epx_hw_cfg,
+ TP_PROTO(struct cdns2_device *pdev, struct cdns2_endpoint *pep),
+ TP_ARGS(pdev, pep)
+);
+
+#endif /* __LINUX_CDNS2_TRACE */
+
+/* This part must be outside header guard. */
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE cdns2-trace
+
+#include <trace/define_trace.h>