Adding upstream version 4.19.249.upstream/4.19.249

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1856 insertions, 0 deletions

diff --git a/drivers/usb/gadget/udc/goku_udc.c b/drivers/usb/gadget/udc/goku_udc.c
new file mode 100644
index 000000000..b706ad303
--- /dev/null
+++ b/drivers/usb/gadget/udc/goku_udc.c
@@ -0,0 +1,1856 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Toshiba TC86C001 ("Goku-S") USB Device Controller driver
+ *
+ * Copyright (C) 2000-2002 Lineo
+ *      by Stuart Lynne, Tom Rushworth, and Bruce Balden
+ * Copyright (C) 2002 Toshiba Corporation
+ * Copyright (C) 2003 MontaVista Software (source@mvista.com)
+ */
+
+/*
+ * This device has ep0 and three semi-configurable bulk/interrupt endpoints.
+ *
+ *  - Endpoint numbering is fixed: ep{1,2,3}-bulk
+ *  - Gadget drivers can choose ep maxpacket (8/16/32/64)
+ *  - Gadget drivers can choose direction (IN, OUT)
+ *  - DMA works with ep1 (OUT transfers) and ep2 (IN transfers).
+ */
+
+// #define	VERBOSE		/* extra debug messages (success too) */
+// #define	USB_TRACE	/* packet-level success messages */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/delay.h>
+#include <linux/ioport.h>
+#include <linux/slab.h>
+#include <linux/errno.h>
+#include <linux/timer.h>
+#include <linux/list.h>
+#include <linux/interrupt.h>
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#include <linux/device.h>
+#include <linux/usb/ch9.h>
+#include <linux/usb/gadget.h>
+#include <linux/prefetch.h>
+
+#include <asm/byteorder.h>
+#include <asm/io.h>
+#include <asm/irq.h>
+#include <asm/unaligned.h>
+
+
+#include "goku_udc.h"
+
+#define	DRIVER_DESC		"TC86C001 USB Device Controller"
+#define	DRIVER_VERSION		"30-Oct 2003"
+
+static const char driver_name [] = "goku_udc";
+static const char driver_desc [] = DRIVER_DESC;
+
+MODULE_AUTHOR("source@mvista.com");
+MODULE_DESCRIPTION(DRIVER_DESC);
+MODULE_LICENSE("GPL");
+
+
+/*
+ * IN dma behaves ok under testing, though the IN-dma abort paths don't
+ * seem to behave quite as expected.  Used by default.
+ *
+ * OUT dma documents design problems handling the common "short packet"
+ * transfer termination policy; it couldn't be enabled by default, even
+ * if the OUT-dma abort problems had a resolution.
+ */
+static unsigned use_dma = 1;
+
+#if 0
+//#include <linux/moduleparam.h>
+/* "modprobe goku_udc use_dma=1" etc
+ *	0 to disable dma
+ *	1 to use IN dma only (normal operation)
+ *	2 to use IN and OUT dma
+ */
+module_param(use_dma, uint, S_IRUGO);
+#endif
+
+/*-------------------------------------------------------------------------*/
+
+static void nuke(struct goku_ep *, int status);
+
+static inline void
+command(struct goku_udc_regs __iomem *regs, int command, unsigned epnum)
+{
+	writel(COMMAND_EP(epnum) | command, &regs->Command);
+	udelay(300);
+}
+
+static int
+goku_ep_enable(struct usb_ep *_ep, const struct usb_endpoint_descriptor *desc)
+{
+	struct goku_udc	*dev;
+	struct goku_ep	*ep;
+	u32		mode;
+	u16		max;
+	unsigned long	flags;
+
+	ep = container_of(_ep, struct goku_ep, ep);
+	if (!_ep || !desc
+			|| desc->bDescriptorType != USB_DT_ENDPOINT)
+		return -EINVAL;
+	dev = ep->dev;
+	if (ep == &dev->ep[0])
+		return -EINVAL;
+	if (!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN)
+		return -ESHUTDOWN;
+	if (ep->num != usb_endpoint_num(desc))
+		return -EINVAL;
+
+	switch (usb_endpoint_type(desc)) {
+	case USB_ENDPOINT_XFER_BULK:
+	case USB_ENDPOINT_XFER_INT:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if ((readl(ep->reg_status) & EPxSTATUS_EP_MASK)
+			!= EPxSTATUS_EP_INVALID)
+		return -EBUSY;
+
+	/* enabling the no-toggle interrupt mode would need an api hook */
+	mode = 0;
+	max = get_unaligned_le16(&desc->wMaxPacketSize);
+	switch (max) {
+	case 64:
+		mode++; /* fall through */
+	case 32:
+		mode++; /* fall through */
+	case 16:
+		mode++; /* fall through */
+	case 8:
+		mode <<= 3;
+		break;
+	default:
+		return -EINVAL;
+	}
+	mode |= 2 << 1;		/* bulk, or intr-with-toggle */
+
+	/* ep1/ep2 dma direction is chosen early; it works in the other
+	 * direction, with pio.  be cautious with out-dma.
+	 */
+	ep->is_in = usb_endpoint_dir_in(desc);
+	if (ep->is_in) {
+		mode |= 1;
+		ep->dma = (use_dma != 0) && (ep->num == UDC_MSTRD_ENDPOINT);
+	} else {
+		ep->dma = (use_dma == 2) && (ep->num == UDC_MSTWR_ENDPOINT);
+		if (ep->dma)
+			DBG(dev, "%s out-dma hides short packets\n",
+				ep->ep.name);
+	}
+
+	spin_lock_irqsave(&ep->dev->lock, flags);
+
+	/* ep1 and ep2 can do double buffering and/or dma */
+	if (ep->num < 3) {
+		struct goku_udc_regs __iomem	*regs = ep->dev->regs;
+		u32				tmp;
+
+		/* double buffer except (for now) with pio in */
+		tmp = ((ep->dma || !ep->is_in)
+				? 0x10	/* double buffered */
+				: 0x11	/* single buffer */
+			) << ep->num;
+		tmp |= readl(&regs->EPxSingle);
+		writel(tmp, &regs->EPxSingle);
+
+		tmp = (ep->dma ? 0x10/*dma*/ : 0x11/*pio*/) << ep->num;
+		tmp |= readl(&regs->EPxBCS);
+		writel(tmp, &regs->EPxBCS);
+	}
+	writel(mode, ep->reg_mode);
+	command(ep->dev->regs, COMMAND_RESET, ep->num);
+	ep->ep.maxpacket = max;
+	ep->stopped = 0;
+	ep->ep.desc = desc;
+	spin_unlock_irqrestore(&ep->dev->lock, flags);
+
+	DBG(dev, "enable %s %s %s maxpacket %u\n", ep->ep.name,
+		ep->is_in ? "IN" : "OUT",
+		ep->dma ? "dma" : "pio",
+		max);
+
+	return 0;
+}
+
+static void ep_reset(struct goku_udc_regs __iomem *regs, struct goku_ep *ep)
+{
+	struct goku_udc		*dev = ep->dev;
+
+	if (regs) {
+		command(regs, COMMAND_INVALID, ep->num);
+		if (ep->num) {
+			if (ep->num == UDC_MSTWR_ENDPOINT)
+				dev->int_enable &= ~(INT_MSTWREND
+							|INT_MSTWRTMOUT);
+			else if (ep->num == UDC_MSTRD_ENDPOINT)
+				dev->int_enable &= ~INT_MSTRDEND;
+			dev->int_enable &= ~INT_EPxDATASET (ep->num);
+		} else
+			dev->int_enable &= ~INT_EP0;
+		writel(dev->int_enable, &regs->int_enable);
+		readl(&regs->int_enable);
+		if (ep->num < 3) {
+			struct goku_udc_regs __iomem	*r = ep->dev->regs;
+			u32				tmp;
+
+			tmp = readl(&r->EPxSingle);
+			tmp &= ~(0x11 << ep->num);
+			writel(tmp, &r->EPxSingle);
+
+			tmp = readl(&r->EPxBCS);
+			tmp &= ~(0x11 << ep->num);
+			writel(tmp, &r->EPxBCS);
+		}
+		/* reset dma in case we're still using it */
+		if (ep->dma) {
+			u32	master;
+
+			master = readl(&regs->dma_master) & MST_RW_BITS;
+			if (ep->num == UDC_MSTWR_ENDPOINT) {
+				master &= ~MST_W_BITS;
+				master |= MST_WR_RESET;
+			} else {
+				master &= ~MST_R_BITS;
+				master |= MST_RD_RESET;
+			}
+			writel(master, &regs->dma_master);
+		}
+	}
+
+	usb_ep_set_maxpacket_limit(&ep->ep, MAX_FIFO_SIZE);
+	ep->ep.desc = NULL;
+	ep->stopped = 1;
+	ep->irqs = 0;
+	ep->dma = 0;
+}
+
+static int goku_ep_disable(struct usb_ep *_ep)
+{
+	struct goku_ep	*ep;
+	struct goku_udc	*dev;
+	unsigned long	flags;
+
+	ep = container_of(_ep, struct goku_ep, ep);
+	if (!_ep || !ep->ep.desc)
+		return -ENODEV;
+	dev = ep->dev;
+	if (dev->ep0state == EP0_SUSPEND)
+		return -EBUSY;
+
+	VDBG(dev, "disable %s\n", _ep->name);
+
+	spin_lock_irqsave(&dev->lock, flags);
+	nuke(ep, -ESHUTDOWN);
+	ep_reset(dev->regs, ep);
+	spin_unlock_irqrestore(&dev->lock, flags);
+
+	return 0;
+}
+
+/*-------------------------------------------------------------------------*/
+
+static struct usb_request *
+goku_alloc_request(struct usb_ep *_ep, gfp_t gfp_flags)
+{
+	struct goku_request	*req;
+
+	if (!_ep)
+		return NULL;
+	req = kzalloc(sizeof *req, gfp_flags);
+	if (!req)
+		return NULL;
+
+	INIT_LIST_HEAD(&req->queue);
+	return &req->req;
+}
+
+static void
+goku_free_request(struct usb_ep *_ep, struct usb_request *_req)
+{
+	struct goku_request	*req;
+
+	if (!_ep || !_req)
+		return;
+
+	req = container_of(_req, struct goku_request, req);
+	WARN_ON(!list_empty(&req->queue));
+	kfree(req);
+}
+
+/*-------------------------------------------------------------------------*/
+
+static void
+done(struct goku_ep *ep, struct goku_request *req, int status)
+{
+	struct goku_udc		*dev;
+	unsigned		stopped = ep->stopped;
+
+	list_del_init(&req->queue);
+
+	if (likely(req->req.status == -EINPROGRESS))
+		req->req.status = status;
+	else
+		status = req->req.status;
+
+	dev = ep->dev;
+
+	if (ep->dma)
+		usb_gadget_unmap_request(&dev->gadget, &req->req, ep->is_in);
+
+#ifndef USB_TRACE
+	if (status && status != -ESHUTDOWN)
+#endif
+		VDBG(dev, "complete %s req %p stat %d len %u/%u\n",
+			ep->ep.name, &req->req, status,
+			req->req.actual, req->req.length);
+
+	/* don't modify queue heads during completion callback */
+	ep->stopped = 1;
+	spin_unlock(&dev->lock);
+	usb_gadget_giveback_request(&ep->ep, &req->req);
+	spin_lock(&dev->lock);
+	ep->stopped = stopped;
+}
+
+/*-------------------------------------------------------------------------*/
+
+static inline int
+write_packet(u32 __iomem *fifo, u8 *buf, struct goku_request *req, unsigned max)
+{
+	unsigned	length, count;
+
+	length = min(req->req.length - req->req.actual, max);
+	req->req.actual += length;
+
+	count = length;
+	while (likely(count--))
+		writel(*buf++, fifo);
+	return length;
+}
+
+// return:  0 = still running, 1 = completed, negative = errno
+static int write_fifo(struct goku_ep *ep, struct goku_request *req)
+{
+	struct goku_udc	*dev = ep->dev;
+	u32		tmp;
+	u8		*buf;
+	unsigned	count;
+	int		is_last;
+
+	tmp = readl(&dev->regs->DataSet);
+	buf = req->req.buf + req->req.actual;
+	prefetch(buf);
+
+	dev = ep->dev;
+	if (unlikely(ep->num == 0 && dev->ep0state != EP0_IN))
+		return -EL2HLT;
+
+	/* NOTE:  just single-buffered PIO-IN for now.  */
+	if (unlikely((tmp & DATASET_A(ep->num)) != 0))
+		return 0;
+
+	/* clear our "packet available" irq */
+	if (ep->num != 0)
+		writel(~INT_EPxDATASET(ep->num), &dev->regs->int_status);
+
+	count = write_packet(ep->reg_fifo, buf, req, ep->ep.maxpacket);
+
+	/* last packet often short (sometimes a zlp, especially on ep0) */
+	if (unlikely(count != ep->ep.maxpacket)) {
+		writel(~(1<<ep->num), &dev->regs->EOP);
+		if (ep->num == 0) {
+			dev->ep[0].stopped = 1;
+			dev->ep0state = EP0_STATUS;
+		}
+		is_last = 1;
+	} else {
+		if (likely(req->req.length != req->req.actual)
+				|| req->req.zero)
+			is_last = 0;
+		else
+			is_last = 1;
+	}
+#if 0		/* printk seemed to trash is_last...*/
+//#ifdef USB_TRACE
+	VDBG(dev, "wrote %s %u bytes%s IN %u left %p\n",
+		ep->ep.name, count, is_last ? "/last" : "",
+		req->req.length - req->req.actual, req);
+#endif
+
+	/* requests complete when all IN data is in the FIFO,
+	 * or sometimes later, if a zlp was needed.
+	 */
+	if (is_last) {
+		done(ep, req, 0);
+		return 1;
+	}
+
+	return 0;
+}
+
+static int read_fifo(struct goku_ep *ep, struct goku_request *req)
+{
+	struct goku_udc_regs __iomem	*regs;
+	u32				size, set;
+	u8				*buf;
+	unsigned			bufferspace, is_short, dbuff;
+
+	regs = ep->dev->regs;
+top:
+	buf = req->req.buf + req->req.actual;
+	prefetchw(buf);
+
+	if (unlikely(ep->num == 0 && ep->dev->ep0state != EP0_OUT))
+		return -EL2HLT;
+
+	dbuff = (ep->num == 1 || ep->num == 2);
+	do {
+		/* ack dataset irq matching the status we'll handle */
+		if (ep->num != 0)
+			writel(~INT_EPxDATASET(ep->num), &regs->int_status);
+
+		set = readl(&regs->DataSet) & DATASET_AB(ep->num);
+		size = readl(&regs->EPxSizeLA[ep->num]);
+		bufferspace = req->req.length - req->req.actual;
+
+		/* usually do nothing without an OUT packet */
+		if (likely(ep->num != 0 || bufferspace != 0)) {
+			if (unlikely(set == 0))
+				break;
+			/* use ep1/ep2 double-buffering for OUT */
+			if (!(size & PACKET_ACTIVE))
+				size = readl(&regs->EPxSizeLB[ep->num]);
+			if (!(size & PACKET_ACTIVE))	/* "can't happen" */
+				break;
+			size &= DATASIZE;	/* EPxSizeH == 0 */
+
+		/* ep0out no-out-data case for set_config, etc */
+		} else
+			size = 0;
+
+		/* read all bytes from this packet */
+		req->req.actual += size;
+		is_short = (size < ep->ep.maxpacket);
+#ifdef USB_TRACE
+		VDBG(ep->dev, "read %s %u bytes%s OUT req %p %u/%u\n",
+			ep->ep.name, size, is_short ? "/S" : "",
+			req, req->req.actual, req->req.length);
+#endif
+		while (likely(size-- != 0)) {
+			u8	byte = (u8) readl(ep->reg_fifo);
+
+			if (unlikely(bufferspace == 0)) {
+				/* this happens when the driver's buffer
+				 * is smaller than what the host sent.
+				 * discard the extra data in this packet.
+				 */
+				if (req->req.status != -EOVERFLOW)
+					DBG(ep->dev, "%s overflow %u\n",
+						ep->ep.name, size);
+				req->req.status = -EOVERFLOW;
+			} else {
+				*buf++ = byte;
+				bufferspace--;
+			}
+		}
+
+		/* completion */
+		if (unlikely(is_short || req->req.actual == req->req.length)) {
+			if (unlikely(ep->num == 0)) {
+				/* non-control endpoints now usable? */
+				if (ep->dev->req_config)
+					writel(ep->dev->configured
+							? USBSTATE_CONFIGURED
+							: 0,
+						&regs->UsbState);
+				/* ep0out status stage */
+				writel(~(1<<0), &regs->EOP);
+				ep->stopped = 1;
+				ep->dev->ep0state = EP0_STATUS;
+			}
+			done(ep, req, 0);
+
+			/* empty the second buffer asap */
+			if (dbuff && !list_empty(&ep->queue)) {
+				req = list_entry(ep->queue.next,
+						struct goku_request, queue);
+				goto top;
+			}
+			return 1;
+		}
+	} while (dbuff);
+	return 0;
+}
+
+static inline void
+pio_irq_enable(struct goku_udc *dev,
+		struct goku_udc_regs __iomem *regs, int epnum)
+{
+	dev->int_enable |= INT_EPxDATASET (epnum);
+	writel(dev->int_enable, &regs->int_enable);
+	/* write may still be posted */
+}
+
+static inline void
+pio_irq_disable(struct goku_udc *dev,
+		struct goku_udc_regs __iomem *regs, int epnum)
+{
+	dev->int_enable &= ~INT_EPxDATASET (epnum);
+	writel(dev->int_enable, &regs->int_enable);
+	/* write may still be posted */
+}
+
+static inline void
+pio_advance(struct goku_ep *ep)
+{
+	struct goku_request	*req;
+
+	if (unlikely(list_empty (&ep->queue)))
+		return;
+	req = list_entry(ep->queue.next, struct goku_request, queue);
+	(ep->is_in ? write_fifo : read_fifo)(ep, req);
+}
+
+
+/*-------------------------------------------------------------------------*/
+
+// return:  0 = q running, 1 = q stopped, negative = errno
+static int start_dma(struct goku_ep *ep, struct goku_request *req)
+{
+	struct goku_udc_regs __iomem	*regs = ep->dev->regs;
+	u32				master;
+	u32				start = req->req.dma;
+	u32				end = start + req->req.length - 1;
+
+	master = readl(&regs->dma_master) & MST_RW_BITS;
+
+	/* re-init the bits affecting IN dma; careful with zlps */
+	if (likely(ep->is_in)) {
+		if (unlikely(master & MST_RD_ENA)) {
+			DBG (ep->dev, "start, IN active dma %03x!!\n",
+				master);
+//			return -EL2HLT;
+		}
+		writel(end, &regs->in_dma_end);
+		writel(start, &regs->in_dma_start);
+
+		master &= ~MST_R_BITS;
+		if (unlikely(req->req.length == 0))
+			master = MST_RD_ENA | MST_RD_EOPB;
+		else if ((req->req.length % ep->ep.maxpacket) != 0
+					|| req->req.zero)
+			master = MST_RD_ENA | MST_EOPB_ENA;
+		else
+			master = MST_RD_ENA | MST_EOPB_DIS;
+
+		ep->dev->int_enable |= INT_MSTRDEND;
+
+	/* Goku DMA-OUT merges short packets, which plays poorly with
+	 * protocols where short packets mark the transfer boundaries.
+	 * The chip supports a nonstandard policy with INT_MSTWRTMOUT,
+	 * ending transfers after 3 SOFs; we don't turn it on.
+	 */
+	} else {
+		if (unlikely(master & MST_WR_ENA)) {
+			DBG (ep->dev, "start, OUT active dma %03x!!\n",
+				master);
+//			return -EL2HLT;
+		}
+		writel(end, &regs->out_dma_end);
+		writel(start, &regs->out_dma_start);
+
+		master &= ~MST_W_BITS;
+		master |= MST_WR_ENA | MST_TIMEOUT_DIS;
+
+		ep->dev->int_enable |= INT_MSTWREND|INT_MSTWRTMOUT;
+	}
+
+	writel(master, &regs->dma_master);
+	writel(ep->dev->int_enable, &regs->int_enable);
+	return 0;
+}
+
+static void dma_advance(struct goku_udc *dev, struct goku_ep *ep)
+{
+	struct goku_request		*req;
+	struct goku_udc_regs __iomem	*regs = ep->dev->regs;
+	u32				master;
+
+	master = readl(&regs->dma_master);
+
+	if (unlikely(list_empty(&ep->queue))) {
+stop:
+		if (ep->is_in)
+			dev->int_enable &= ~INT_MSTRDEND;
+		else
+			dev->int_enable &= ~(INT_MSTWREND|INT_MSTWRTMOUT);
+		writel(dev->int_enable, &regs->int_enable);
+		return;
+	}
+	req = list_entry(ep->queue.next, struct goku_request, queue);
+
+	/* normal hw dma completion (not abort) */
+	if (likely(ep->is_in)) {
+		if (unlikely(master & MST_RD_ENA))
+			return;
+		req->req.actual = readl(&regs->in_dma_current);
+	} else {
+		if (unlikely(master & MST_WR_ENA))
+			return;
+
+		/* hardware merges short packets, and also hides packet
+		 * overruns.  a partial packet MAY be in the fifo here.
+		 */
+		req->req.actual = readl(&regs->out_dma_current);
+	}
+	req->req.actual -= req->req.dma;
+	req->req.actual++;
+
+#ifdef USB_TRACE
+	VDBG(dev, "done %s %s dma, %u/%u bytes, req %p\n",
+		ep->ep.name, ep->is_in ? "IN" : "OUT",
+		req->req.actual, req->req.length, req);
+#endif
+	done(ep, req, 0);
+	if (list_empty(&ep->queue))
+		goto stop;
+	req = list_entry(ep->queue.next, struct goku_request, queue);
+	(void) start_dma(ep, req);
+}
+
+static void abort_dma(struct goku_ep *ep, int status)
+{
+	struct goku_udc_regs __iomem	*regs = ep->dev->regs;
+	struct goku_request		*req;
+	u32				curr, master;
+
+	/* NAK future host requests, hoping the implicit delay lets the
+	 * dma engine finish reading (or writing) its latest packet and
+	 * empty the dma buffer (up to 16 bytes).
+	 *
+	 * This avoids needing to clean up a partial packet in the fifo;
+	 * we can't do that for IN without side effects to HALT and TOGGLE.
+	 */
+	command(regs, COMMAND_FIFO_DISABLE, ep->num);
+	req = list_entry(ep->queue.next, struct goku_request, queue);
+	master = readl(&regs->dma_master) & MST_RW_BITS;
+
+	/* FIXME using these resets isn't usably documented. this may
+	 * not work unless it's followed by disabling the endpoint.
+	 *
+	 * FIXME the OUT reset path doesn't even behave consistently.
+	 */
+	if (ep->is_in) {
+		if (unlikely((readl(&regs->dma_master) & MST_RD_ENA) == 0))
+			goto finished;
+		curr = readl(&regs->in_dma_current);
+
+		writel(curr, &regs->in_dma_end);
+		writel(curr, &regs->in_dma_start);
+
+		master &= ~MST_R_BITS;
+		master |= MST_RD_RESET;
+		writel(master, &regs->dma_master);
+
+		if (readl(&regs->dma_master) & MST_RD_ENA)
+			DBG(ep->dev, "IN dma active after reset!\n");
+
+	} else {
+		if (unlikely((readl(&regs->dma_master) & MST_WR_ENA) == 0))
+			goto finished;
+		curr = readl(&regs->out_dma_current);
+
+		writel(curr, &regs->out_dma_end);
+		writel(curr, &regs->out_dma_start);
+
+		master &= ~MST_W_BITS;
+		master |= MST_WR_RESET;
+		writel(master, &regs->dma_master);
+
+		if (readl(&regs->dma_master) & MST_WR_ENA)
+			DBG(ep->dev, "OUT dma active after reset!\n");
+	}
+	req->req.actual = (curr - req->req.dma) + 1;
+	req->req.status = status;
+
+	VDBG(ep->dev, "%s %s %s %d/%d\n", __func__, ep->ep.name,
+		ep->is_in ? "IN" : "OUT",
+		req->req.actual, req->req.length);
+
+	command(regs, COMMAND_FIFO_ENABLE, ep->num);
+
+	return;
+
+finished:
+	/* dma already completed; no abort needed */
+	command(regs, COMMAND_FIFO_ENABLE, ep->num);
+	req->req.actual = req->req.length;
+	req->req.status = 0;
+}
+
+/*-------------------------------------------------------------------------*/
+
+static int
+goku_queue(struct usb_ep *_ep, struct usb_request *_req, gfp_t gfp_flags)
+{
+	struct goku_request	*req;
+	struct goku_ep		*ep;
+	struct goku_udc		*dev;
+	unsigned long		flags;
+	int			status;
+
+	/* always require a cpu-view buffer so pio works */
+	req = container_of(_req, struct goku_request, req);
+	if (unlikely(!_req || !_req->complete
+			|| !_req->buf || !list_empty(&req->queue)))
+		return -EINVAL;
+	ep = container_of(_ep, struct goku_ep, ep);
+	if (unlikely(!_ep || (!ep->ep.desc && ep->num != 0)))
+		return -EINVAL;
+	dev = ep->dev;
+	if (unlikely(!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN))
+		return -ESHUTDOWN;
+
+	/* can't touch registers when suspended */
+	if (dev->ep0state == EP0_SUSPEND)
+		return -EBUSY;
+
+	/* set up dma mapping in case the caller didn't */
+	if (ep->dma) {
+		status = usb_gadget_map_request(&dev->gadget, &req->req,
+				ep->is_in);
+		if (status)
+			return status;
+	}
+
+#ifdef USB_TRACE
+	VDBG(dev, "%s queue req %p, len %u buf %p\n",
+			_ep->name, _req, _req->length, _req->buf);
+#endif
+
+	spin_lock_irqsave(&dev->lock, flags);
+
+	_req->status = -EINPROGRESS;
+	_req->actual = 0;
+
+	/* for ep0 IN without premature status, zlp is required and
+	 * writing EOP starts the status stage (OUT).
+	 */
+	if (unlikely(ep->num == 0 && ep->is_in))
+		_req->zero = 1;
+
+	/* kickstart this i/o queue? */
+	status = 0;
+	if (list_empty(&ep->queue) && likely(!ep->stopped)) {
+		/* dma:  done after dma completion IRQ (or error)
+		 * pio:  done after last fifo operation
+		 */
+		if (ep->dma)
+			status = start_dma(ep, req);
+		else
+			status = (ep->is_in ? write_fifo : read_fifo)(ep, req);
+
+		if (unlikely(status != 0)) {
+			if (status > 0)
+				status = 0;
+			req = NULL;
+		}
+
+	} /* else pio or dma irq handler advances the queue. */
+
+	if (likely(req != NULL))
+		list_add_tail(&req->queue, &ep->queue);
+
+	if (likely(!list_empty(&ep->queue))
+			&& likely(ep->num != 0)
+			&& !ep->dma
+			&& !(dev->int_enable & INT_EPxDATASET (ep->num)))
+		pio_irq_enable(dev, dev->regs, ep->num);
+
+	spin_unlock_irqrestore(&dev->lock, flags);
+
+	/* pci writes may still be posted */
+	return status;
+}
+
+/* dequeue ALL requests */
+static void nuke(struct goku_ep *ep, int status)
+{
+	struct goku_request	*req;
+
+	ep->stopped = 1;
+	if (list_empty(&ep->queue))
+		return;
+	if (ep->dma)
+		abort_dma(ep, status);
+	while (!list_empty(&ep->queue)) {
+		req = list_entry(ep->queue.next, struct goku_request, queue);
+		done(ep, req, status);
+	}
+}
+
+/* dequeue JUST ONE request */
+static int goku_dequeue(struct usb_ep *_ep, struct usb_request *_req)
+{
+	struct goku_request	*req;
+	struct goku_ep		*ep;
+	struct goku_udc		*dev;
+	unsigned long		flags;
+
+	ep = container_of(_ep, struct goku_ep, ep);
+	if (!_ep || !_req || (!ep->ep.desc && ep->num != 0))
+		return -EINVAL;
+	dev = ep->dev;
+	if (!dev->driver)
+		return -ESHUTDOWN;
+
+	/* we can't touch (dma) registers when suspended */
+	if (dev->ep0state == EP0_SUSPEND)
+		return -EBUSY;
+
+	VDBG(dev, "%s %s %s %s %p\n", __func__, _ep->name,
+		ep->is_in ? "IN" : "OUT",
+		ep->dma ? "dma" : "pio",
+		_req);
+
+	spin_lock_irqsave(&dev->lock, flags);
+
+	/* make sure it's actually queued on this endpoint */
+	list_for_each_entry (req, &ep->queue, queue) {
+		if (&req->req == _req)
+			break;
+	}
+	if (&req->req != _req) {
+		spin_unlock_irqrestore (&dev->lock, flags);
+		return -EINVAL;
+	}
+
+	if (ep->dma && ep->queue.next == &req->queue && !ep->stopped) {
+		abort_dma(ep, -ECONNRESET);
+		done(ep, req, -ECONNRESET);
+		dma_advance(dev, ep);
+	} else if (!list_empty(&req->queue))
+		done(ep, req, -ECONNRESET);
+	else
+		req = NULL;
+	spin_unlock_irqrestore(&dev->lock, flags);
+
+	return req ? 0 : -EOPNOTSUPP;
+}
+
+/*-------------------------------------------------------------------------*/
+
+static void goku_clear_halt(struct goku_ep *ep)
+{
+	// assert (ep->num !=0)
+	VDBG(ep->dev, "%s clear halt\n", ep->ep.name);
+	command(ep->dev->regs, COMMAND_SETDATA0, ep->num);
+	command(ep->dev->regs, COMMAND_STALL_CLEAR, ep->num);
+	if (ep->stopped) {
+		ep->stopped = 0;
+		if (ep->dma) {
+			struct goku_request	*req;
+
+			if (list_empty(&ep->queue))
+				return;
+			req = list_entry(ep->queue.next, struct goku_request,
+						queue);
+			(void) start_dma(ep, req);
+		} else
+			pio_advance(ep);
+	}
+}
+
+static int goku_set_halt(struct usb_ep *_ep, int value)
+{
+	struct goku_ep	*ep;
+	unsigned long	flags;
+	int		retval = 0;
+
+	if (!_ep)
+		return -ENODEV;
+	ep = container_of (_ep, struct goku_ep, ep);
+
+	if (ep->num == 0) {
+		if (value) {
+			ep->dev->ep0state = EP0_STALL;
+			ep->dev->ep[0].stopped = 1;
+		} else
+			return -EINVAL;
+
+	/* don't change EPxSTATUS_EP_INVALID to READY */
+	} else if (!ep->ep.desc) {
+		DBG(ep->dev, "%s %s inactive?\n", __func__, ep->ep.name);
+		return -EINVAL;
+	}
+
+	spin_lock_irqsave(&ep->dev->lock, flags);
+	if (!list_empty(&ep->queue))
+		retval = -EAGAIN;
+	else if (ep->is_in && value
+			/* data in (either) packet buffer? */
+			&& (readl(&ep->dev->regs->DataSet)
+					& DATASET_AB(ep->num)))
+		retval = -EAGAIN;
+	else if (!value)
+		goku_clear_halt(ep);
+	else {
+		ep->stopped = 1;
+		VDBG(ep->dev, "%s set halt\n", ep->ep.name);
+		command(ep->dev->regs, COMMAND_STALL, ep->num);
+		readl(ep->reg_status);
+	}
+	spin_unlock_irqrestore(&ep->dev->lock, flags);
+	return retval;
+}
+
+static int goku_fifo_status(struct usb_ep *_ep)
+{
+	struct goku_ep			*ep;
+	struct goku_udc_regs __iomem	*regs;
+	u32				size;
+
+	if (!_ep)
+		return -ENODEV;
+	ep = container_of(_ep, struct goku_ep, ep);
+
+	/* size is only reported sanely for OUT */
+	if (ep->is_in)
+		return -EOPNOTSUPP;
+
+	/* ignores 16-byte dma buffer; SizeH == 0 */
+	regs = ep->dev->regs;
+	size = readl(&regs->EPxSizeLA[ep->num]) & DATASIZE;
+	size += readl(&regs->EPxSizeLB[ep->num]) & DATASIZE;
+	VDBG(ep->dev, "%s %s %u\n", __func__, ep->ep.name, size);
+	return size;
+}
+
+static void goku_fifo_flush(struct usb_ep *_ep)
+{
+	struct goku_ep			*ep;
+	struct goku_udc_regs __iomem	*regs;
+	u32				size;
+
+	if (!_ep)
+		return;
+	ep = container_of(_ep, struct goku_ep, ep);
+	VDBG(ep->dev, "%s %s\n", __func__, ep->ep.name);
+
+	/* don't change EPxSTATUS_EP_INVALID to READY */
+	if (!ep->ep.desc && ep->num != 0) {
+		DBG(ep->dev, "%s %s inactive?\n", __func__, ep->ep.name);
+		return;
+	}
+
+	regs = ep->dev->regs;
+	size = readl(&regs->EPxSizeLA[ep->num]);
+	size &= DATASIZE;
+
+	/* Non-desirable behavior:  FIFO_CLEAR also clears the
+	 * endpoint halt feature.  For OUT, we _could_ just read
+	 * the bytes out (PIO, if !ep->dma); for in, no choice.
+	 */
+	if (size)
+		command(regs, COMMAND_FIFO_CLEAR, ep->num);
+}
+
+static const struct usb_ep_ops goku_ep_ops = {
+	.enable		= goku_ep_enable,
+	.disable	= goku_ep_disable,
+
+	.alloc_request	= goku_alloc_request,
+	.free_request	= goku_free_request,
+
+	.queue		= goku_queue,
+	.dequeue	= goku_dequeue,
+
+	.set_halt	= goku_set_halt,
+	.fifo_status	= goku_fifo_status,
+	.fifo_flush	= goku_fifo_flush,
+};
+
+/*-------------------------------------------------------------------------*/
+
+static int goku_get_frame(struct usb_gadget *_gadget)
+{
+	return -EOPNOTSUPP;
+}
+
+static struct usb_ep *goku_match_ep(struct usb_gadget *g,
+		struct usb_endpoint_descriptor *desc,
+		struct usb_ss_ep_comp_descriptor *ep_comp)
+{
+	struct goku_udc	*dev = to_goku_udc(g);
+	struct usb_ep *ep;
+
+	switch (usb_endpoint_type(desc)) {
+	case USB_ENDPOINT_XFER_INT:
+		/* single buffering is enough */
+		ep = &dev->ep[3].ep;
+		if (usb_gadget_ep_match_desc(g, ep, desc, ep_comp))
+			return ep;
+		break;
+	case USB_ENDPOINT_XFER_BULK:
+		if (usb_endpoint_dir_in(desc)) {
+			/* DMA may be available */
+			ep = &dev->ep[2].ep;
+			if (usb_gadget_ep_match_desc(g, ep, desc, ep_comp))
+				return ep;
+		}
+		break;
+	default:
+		/* nothing */ ;
+	}
+
+	return NULL;
+}
+
+static int goku_udc_start(struct usb_gadget *g,
+		struct usb_gadget_driver *driver);
+static int goku_udc_stop(struct usb_gadget *g);
+
+static const struct usb_gadget_ops goku_ops = {
+	.get_frame	= goku_get_frame,
+	.udc_start	= goku_udc_start,
+	.udc_stop	= goku_udc_stop,
+	.match_ep	= goku_match_ep,
+	// no remote wakeup
+	// not selfpowered
+};
+
+/*-------------------------------------------------------------------------*/
+
+static inline const char *dmastr(void)
+{
+	if (use_dma == 0)
+		return "(dma disabled)";
+	else if (use_dma == 2)
+		return "(dma IN and OUT)";
+	else
+		return "(dma IN)";
+}
+
+#ifdef CONFIG_USB_GADGET_DEBUG_FILES
+
+static const char proc_node_name [] = "driver/udc";
+
+#define FOURBITS "%s%s%s%s"
+#define EIGHTBITS FOURBITS FOURBITS
+
+static void dump_intmask(struct seq_file *m, const char *label, u32 mask)
+{
+	/* int_status is the same format ... */
+	seq_printf(m, "%s %05X =" FOURBITS EIGHTBITS EIGHTBITS "\n",
+		   label, mask,
+		   (mask & INT_PWRDETECT) ? " power" : "",
+		   (mask & INT_SYSERROR) ? " sys" : "",
+		   (mask & INT_MSTRDEND) ? " in-dma" : "",
+		   (mask & INT_MSTWRTMOUT) ? " wrtmo" : "",
+
+		   (mask & INT_MSTWREND) ? " out-dma" : "",
+		   (mask & INT_MSTWRSET) ? " wrset" : "",
+		   (mask & INT_ERR) ? " err" : "",
+		   (mask & INT_SOF) ? " sof" : "",
+
+		   (mask & INT_EP3NAK) ? " ep3nak" : "",
+		   (mask & INT_EP2NAK) ? " ep2nak" : "",
+		   (mask & INT_EP1NAK) ? " ep1nak" : "",
+		   (mask & INT_EP3DATASET) ? " ep3" : "",
+
+		   (mask & INT_EP2DATASET) ? " ep2" : "",
+		   (mask & INT_EP1DATASET) ? " ep1" : "",
+		   (mask & INT_STATUSNAK) ? " ep0snak" : "",
+		   (mask & INT_STATUS) ? " ep0status" : "",
+
+		   (mask & INT_SETUP) ? " setup" : "",
+		   (mask & INT_ENDPOINT0) ? " ep0" : "",
+		   (mask & INT_USBRESET) ? " reset" : "",
+		   (mask & INT_SUSPEND) ? " suspend" : "");
+}
+
+static const char *udc_ep_state(enum ep0state state)
+{
+	switch (state) {
+	case EP0_DISCONNECT:
+		return "ep0_disconnect";
+	case EP0_IDLE:
+		return "ep0_idle";
+	case EP0_IN:
+		return "ep0_in";
+	case EP0_OUT:
+		return "ep0_out";
+	case EP0_STATUS:
+		return "ep0_status";
+	case EP0_STALL:
+		return "ep0_stall";
+	case EP0_SUSPEND:
+		return "ep0_suspend";
+	}
+
+	return "ep0_?";
+}
+
+static const char *udc_ep_status(u32 status)
+{
+	switch (status & EPxSTATUS_EP_MASK) {
+	case EPxSTATUS_EP_READY:
+		return "ready";
+	case EPxSTATUS_EP_DATAIN:
+		return "packet";
+	case EPxSTATUS_EP_FULL:
+		return "full";
+	case EPxSTATUS_EP_TX_ERR:	/* host will retry */
+		return "tx_err";
+	case EPxSTATUS_EP_RX_ERR:
+		return "rx_err";
+	case EPxSTATUS_EP_BUSY:		/* ep0 only */
+		return "busy";
+	case EPxSTATUS_EP_STALL:
+		return "stall";
+	case EPxSTATUS_EP_INVALID:	/* these "can't happen" */
+		return "invalid";
+	}
+
+	return "?";
+}
+
+static int udc_proc_read(struct seq_file *m, void *v)
+{
+	struct goku_udc			*dev = m->private;
+	struct goku_udc_regs __iomem	*regs = dev->regs;
+	unsigned long			flags;
+	int				i, is_usb_connected;
+	u32				tmp;
+
+	local_irq_save(flags);
+
+	/* basic device status */
+	tmp = readl(&regs->power_detect);
+	is_usb_connected = tmp & PW_DETECT;
+	seq_printf(m,
+		   "%s - %s\n"
+		   "%s version: %s %s\n"
+		   "Gadget driver: %s\n"
+		   "Host %s, %s\n"
+		   "\n",
+		   pci_name(dev->pdev), driver_desc,
+		   driver_name, DRIVER_VERSION, dmastr(),
+		   dev->driver ? dev->driver->driver.name : "(none)",
+		   is_usb_connected
+			   ? ((tmp & PW_PULLUP) ? "full speed" : "powered")
+			   : "disconnected",
+		   udc_ep_state(dev->ep0state));
+
+	dump_intmask(m, "int_status", readl(&regs->int_status));
+	dump_intmask(m, "int_enable", readl(&regs->int_enable));
+
+	if (!is_usb_connected || !dev->driver || (tmp & PW_PULLUP) == 0)
+		goto done;
+
+	/* registers for (active) device and ep0 */
+	seq_printf(m, "\nirqs %lu\ndataset %02x single.bcs %02x.%02x state %x addr %u\n",
+		   dev->irqs, readl(&regs->DataSet),
+		   readl(&regs->EPxSingle), readl(&regs->EPxBCS),
+		   readl(&regs->UsbState),
+		   readl(&regs->address));
+	if (seq_has_overflowed(m))
+		goto done;
+
+	tmp = readl(&regs->dma_master);
+	seq_printf(m, "dma %03X =" EIGHTBITS "%s %s\n",
+		   tmp,
+		   (tmp & MST_EOPB_DIS) ? " eopb-" : "",
+		   (tmp & MST_EOPB_ENA) ? " eopb+" : "",
+		   (tmp & MST_TIMEOUT_DIS) ? " tmo-" : "",
+		   (tmp & MST_TIMEOUT_ENA) ? " tmo+" : "",
+
+		   (tmp & MST_RD_EOPB) ? " eopb" : "",
+		   (tmp & MST_RD_RESET) ? " in_reset" : "",
+		   (tmp & MST_WR_RESET) ? " out_reset" : "",
+		   (tmp & MST_RD_ENA) ? " IN" : "",
+
+		   (tmp & MST_WR_ENA) ? " OUT" : "",
+		   (tmp & MST_CONNECTION) ? "ep1in/ep2out" : "ep1out/ep2in");
+	if (seq_has_overflowed(m))
+		goto done;
+
+	/* dump endpoint queues */
+	for (i = 0; i < 4; i++) {
+		struct goku_ep		*ep = &dev->ep [i];
+		struct goku_request	*req;
+
+		if (i && !ep->ep.desc)
+			continue;
+
+		tmp = readl(ep->reg_status);
+		seq_printf(m, "%s %s max %u %s, irqs %lu, status %02x (%s) " FOURBITS "\n",
+			   ep->ep.name,
+			   ep->is_in ? "in" : "out",
+			   ep->ep.maxpacket,
+			   ep->dma ? "dma" : "pio",
+			   ep->irqs,
+			   tmp, udc_ep_status(tmp),
+			   (tmp & EPxSTATUS_TOGGLE) ? "data1" : "data0",
+			   (tmp & EPxSTATUS_SUSPEND) ? " suspend" : "",
+			   (tmp & EPxSTATUS_FIFO_DISABLE) ? " disable" : "",
+			   (tmp & EPxSTATUS_STAGE_ERROR) ? " ep0stat" : "");
+		if (seq_has_overflowed(m))
+			goto done;
+
+		if (list_empty(&ep->queue)) {
+			seq_puts(m, "\t(nothing queued)\n");
+			if (seq_has_overflowed(m))
+				goto done;
+			continue;
+		}
+		list_for_each_entry(req, &ep->queue, queue) {
+			if (ep->dma && req->queue.prev == &ep->queue) {
+				if (i == UDC_MSTRD_ENDPOINT)
+					tmp = readl(&regs->in_dma_current);
+				else
+					tmp = readl(&regs->out_dma_current);
+				tmp -= req->req.dma;
+				tmp++;
+			} else
+				tmp = req->req.actual;
+
+			seq_printf(m, "\treq %p len %u/%u buf %p\n",
+				   &req->req, tmp, req->req.length,
+				   req->req.buf);
+			if (seq_has_overflowed(m))
+				goto done;
+		}
+	}
+
+done:
+	local_irq_restore(flags);
+	return 0;
+}
+#endif	/* CONFIG_USB_GADGET_DEBUG_FILES */
+
+/*-------------------------------------------------------------------------*/
+
+static void udc_reinit (struct goku_udc *dev)
+{
+	static char *names [] = { "ep0", "ep1-bulk", "ep2-bulk", "ep3-bulk" };
+
+	unsigned i;
+
+	INIT_LIST_HEAD (&dev->gadget.ep_list);
+	dev->gadget.ep0 = &dev->ep [0].ep;
+	dev->gadget.speed = USB_SPEED_UNKNOWN;
+	dev->ep0state = EP0_DISCONNECT;
+	dev->irqs = 0;
+
+	for (i = 0; i < 4; i++) {
+		struct goku_ep	*ep = &dev->ep[i];
+
+		ep->num = i;
+		ep->ep.name = names[i];
+		ep->reg_fifo = &dev->regs->ep_fifo [i];
+		ep->reg_status = &dev->regs->ep_status [i];
+		ep->reg_mode = &dev->regs->ep_mode[i];
+
+		ep->ep.ops = &goku_ep_ops;
+		list_add_tail (&ep->ep.ep_list, &dev->gadget.ep_list);
+		ep->dev = dev;
+		INIT_LIST_HEAD (&ep->queue);
+
+		ep_reset(NULL, ep);
+
+		if (i == 0)
+			ep->ep.caps.type_control = true;
+		else
+			ep->ep.caps.type_bulk = true;
+
+		ep->ep.caps.dir_in = true;
+		ep->ep.caps.dir_out = true;
+	}
+
+	dev->ep[0].reg_mode = NULL;
+	usb_ep_set_maxpacket_limit(&dev->ep[0].ep, MAX_EP0_SIZE);
+	list_del_init (&dev->ep[0].ep.ep_list);
+}
+
+static void udc_reset(struct goku_udc *dev)
+{
+	struct goku_udc_regs __iomem	*regs = dev->regs;
+
+	writel(0, &regs->power_detect);
+	writel(0, &regs->int_enable);
+	readl(&regs->int_enable);
+	dev->int_enable = 0;
+
+	/* deassert reset, leave USB D+ at hi-Z (no pullup)
+	 * don't let INT_PWRDETECT sequence begin
+	 */
+	udelay(250);
+	writel(PW_RESETB, &regs->power_detect);
+	readl(&regs->int_enable);
+}
+
+static void ep0_start(struct goku_udc *dev)
+{
+	struct goku_udc_regs __iomem	*regs = dev->regs;
+	unsigned			i;
+
+	VDBG(dev, "%s\n", __func__);
+
+	udc_reset(dev);
+	udc_reinit (dev);
+	//writel(MST_EOPB_ENA | MST_TIMEOUT_ENA, &regs->dma_master);
+
+	/* hw handles set_address, set_feature, get_status; maybe more */
+	writel(   G_REQMODE_SET_INTF | G_REQMODE_GET_INTF
+		| G_REQMODE_SET_CONF | G_REQMODE_GET_CONF
+		| G_REQMODE_GET_DESC
+		| G_REQMODE_CLEAR_FEAT
+		, &regs->reqmode);
+
+	for (i = 0; i < 4; i++)
+		dev->ep[i].irqs = 0;
+
+	/* can't modify descriptors after writing UsbReady */
+	for (i = 0; i < DESC_LEN; i++)
+		writel(0, &regs->descriptors[i]);
+	writel(0, &regs->UsbReady);
+
+	/* expect ep0 requests when the host drops reset */
+	writel(PW_RESETB | PW_PULLUP, &regs->power_detect);
+	dev->int_enable = INT_DEVWIDE | INT_EP0;
+	writel(dev->int_enable, &dev->regs->int_enable);
+	readl(&regs->int_enable);
+	dev->gadget.speed = USB_SPEED_FULL;
+	dev->ep0state = EP0_IDLE;
+}
+
+static void udc_enable(struct goku_udc *dev)
+{
+	/* start enumeration now, or after power detect irq */
+	if (readl(&dev->regs->power_detect) & PW_DETECT)
+		ep0_start(dev);
+	else {
+		DBG(dev, "%s\n", __func__);
+		dev->int_enable = INT_PWRDETECT;
+		writel(dev->int_enable, &dev->regs->int_enable);
+	}
+}
+
+/*-------------------------------------------------------------------------*/
+
+/* keeping it simple:
+ * - one bus driver, initted first;
+ * - one function driver, initted second
+ */
+
+/* when a driver is successfully registered, it will receive
+ * control requests including set_configuration(), which enables
+ * non-control requests.  then usb traffic follows until a
+ * disconnect is reported.  then a host may connect again, or
+ * the driver might get unbound.
+ */
+static int goku_udc_start(struct usb_gadget *g,
+		struct usb_gadget_driver *driver)
+{
+	struct goku_udc	*dev = to_goku_udc(g);
+
+	/* hook up the driver */
+	driver->driver.bus = NULL;
+	dev->driver = driver;
+
+	/*
+	 * then enable host detection and ep0; and we're ready
+	 * for set_configuration as well as eventual disconnect.
+	 */
+	udc_enable(dev);
+
+	return 0;
+}
+
+static void stop_activity(struct goku_udc *dev)
+{
+	unsigned	i;
+
+	DBG (dev, "%s\n", __func__);
+
+	/* disconnect gadget driver after quiesceing hw and the driver */
+	udc_reset (dev);
+	for (i = 0; i < 4; i++)
+		nuke(&dev->ep [i], -ESHUTDOWN);
+
+	if (dev->driver)
+		udc_enable(dev);
+}
+
+static int goku_udc_stop(struct usb_gadget *g)
+{
+	struct goku_udc	*dev = to_goku_udc(g);
+	unsigned long	flags;
+
+	spin_lock_irqsave(&dev->lock, flags);
+	dev->driver = NULL;
+	stop_activity(dev);
+	spin_unlock_irqrestore(&dev->lock, flags);
+
+	return 0;
+}
+
+/*-------------------------------------------------------------------------*/
+
+static void ep0_setup(struct goku_udc *dev)
+{
+	struct goku_udc_regs __iomem	*regs = dev->regs;
+	struct usb_ctrlrequest		ctrl;
+	int				tmp;
+
+	/* read SETUP packet and enter DATA stage */
+	ctrl.bRequestType = readl(&regs->bRequestType);
+	ctrl.bRequest = readl(&regs->bRequest);
+	ctrl.wValue  = cpu_to_le16((readl(&regs->wValueH)  << 8)
+					| readl(&regs->wValueL));
+	ctrl.wIndex  = cpu_to_le16((readl(&regs->wIndexH)  << 8)
+					| readl(&regs->wIndexL));
+	ctrl.wLength = cpu_to_le16((readl(&regs->wLengthH) << 8)
+					| readl(&regs->wLengthL));
+	writel(0, &regs->SetupRecv);
+
+	nuke(&dev->ep[0], 0);
+	dev->ep[0].stopped = 0;
+	if (likely(ctrl.bRequestType & USB_DIR_IN)) {
+		dev->ep[0].is_in = 1;
+		dev->ep0state = EP0_IN;
+		/* detect early status stages */
+		writel(ICONTROL_STATUSNAK, &dev->regs->IntControl);
+	} else {
+		dev->ep[0].is_in = 0;
+		dev->ep0state = EP0_OUT;
+
+		/* NOTE:  CLEAR_FEATURE is done in software so that we can
+		 * synchronize transfer restarts after bulk IN stalls.  data
+		 * won't even enter the fifo until the halt is cleared.
+		 */
+		switch (ctrl.bRequest) {
+		case USB_REQ_CLEAR_FEATURE:
+			switch (ctrl.bRequestType) {
+			case USB_RECIP_ENDPOINT:
+				tmp = le16_to_cpu(ctrl.wIndex) & 0x0f;
+				/* active endpoint */
+				if (tmp > 3 ||
+				    (!dev->ep[tmp].ep.desc && tmp != 0))
+					goto stall;
+				if (ctrl.wIndex & cpu_to_le16(
+						USB_DIR_IN)) {
+					if (!dev->ep[tmp].is_in)
+						goto stall;
+				} else {
+					if (dev->ep[tmp].is_in)
+						goto stall;
+				}
+				if (ctrl.wValue != cpu_to_le16(
+						USB_ENDPOINT_HALT))
+					goto stall;
+				if (tmp)
+					goku_clear_halt(&dev->ep[tmp]);
+succeed:
+				/* start ep0out status stage */
+				writel(~(1<<0), &regs->EOP);
+				dev->ep[0].stopped = 1;
+				dev->ep0state = EP0_STATUS;
+				return;
+			case USB_RECIP_DEVICE:
+				/* device remote wakeup: always clear */
+				if (ctrl.wValue != cpu_to_le16(1))
+					goto stall;
+				VDBG(dev, "clear dev remote wakeup\n");
+				goto succeed;
+			case USB_RECIP_INTERFACE:
+				goto stall;
+			default:		/* pass to gadget driver */
+				break;
+			}
+			break;
+		default:
+			break;
+		}
+	}
+
+#ifdef USB_TRACE
+	VDBG(dev, "SETUP %02x.%02x v%04x i%04x l%04x\n",
+		ctrl.bRequestType, ctrl.bRequest,
+		le16_to_cpu(ctrl.wValue), le16_to_cpu(ctrl.wIndex),
+		le16_to_cpu(ctrl.wLength));
+#endif
+
+	/* hw wants to know when we're configured (or not) */
+	dev->req_config = (ctrl.bRequest == USB_REQ_SET_CONFIGURATION
+				&& ctrl.bRequestType == USB_RECIP_DEVICE);
+	if (unlikely(dev->req_config))
+		dev->configured = (ctrl.wValue != cpu_to_le16(0));
+
+	/* delegate everything to the gadget driver.
+	 * it may respond after this irq handler returns.
+	 */
+	spin_unlock (&dev->lock);
+	tmp = dev->driver->setup(&dev->gadget, &ctrl);
+	spin_lock (&dev->lock);
+	if (unlikely(tmp < 0)) {
+stall:
+#ifdef USB_TRACE
+		VDBG(dev, "req %02x.%02x protocol STALL; err %d\n",
+				ctrl.bRequestType, ctrl.bRequest, tmp);
+#endif
+		command(regs, COMMAND_STALL, 0);
+		dev->ep[0].stopped = 1;
+		dev->ep0state = EP0_STALL;
+	}
+
+	/* expect at least one data or status stage irq */
+}
+
+#define ACK(irqbit) { \
+		stat &= ~irqbit; \
+		writel(~irqbit, &regs->int_status); \
+		handled = 1; \
+		}
+
+static irqreturn_t goku_irq(int irq, void *_dev)
+{
+	struct goku_udc			*dev = _dev;
+	struct goku_udc_regs __iomem	*regs = dev->regs;
+	struct goku_ep			*ep;
+	u32				stat, handled = 0;
+	unsigned			i, rescans = 5;
+
+	spin_lock(&dev->lock);
+
+rescan:
+	stat = readl(&regs->int_status) & dev->int_enable;
+        if (!stat)
+		goto done;
+	dev->irqs++;
+
+	/* device-wide irqs */
+	if (unlikely(stat & INT_DEVWIDE)) {
+		if (stat & INT_SYSERROR) {
+			ERROR(dev, "system error\n");
+			stop_activity(dev);
+			stat = 0;
+			handled = 1;
+			// FIXME have a neater way to prevent re-enumeration
+			dev->driver = NULL;
+			goto done;
+		}
+		if (stat & INT_PWRDETECT) {
+			writel(~stat, &regs->int_status);
+			if (readl(&dev->regs->power_detect) & PW_DETECT) {
+				VDBG(dev, "connect\n");
+				ep0_start(dev);
+			} else {
+				DBG(dev, "disconnect\n");
+				if (dev->gadget.speed == USB_SPEED_FULL)
+					stop_activity(dev);
+				dev->ep0state = EP0_DISCONNECT;
+				dev->int_enable = INT_DEVWIDE;
+				writel(dev->int_enable, &dev->regs->int_enable);
+			}
+			stat = 0;
+			handled = 1;
+			goto done;
+		}
+		if (stat & INT_SUSPEND) {
+			ACK(INT_SUSPEND);
+			if (readl(&regs->ep_status[0]) & EPxSTATUS_SUSPEND) {
+				switch (dev->ep0state) {
+				case EP0_DISCONNECT:
+				case EP0_SUSPEND:
+					goto pm_next;
+				default:
+					break;
+				}
+				DBG(dev, "USB suspend\n");
+				dev->ep0state = EP0_SUSPEND;
+				if (dev->gadget.speed != USB_SPEED_UNKNOWN
+						&& dev->driver
+						&& dev->driver->suspend) {
+					spin_unlock(&dev->lock);
+					dev->driver->suspend(&dev->gadget);
+					spin_lock(&dev->lock);
+				}
+			} else {
+				if (dev->ep0state != EP0_SUSPEND) {
+					DBG(dev, "bogus USB resume %d\n",
+						dev->ep0state);
+					goto pm_next;
+				}
+				DBG(dev, "USB resume\n");
+				dev->ep0state = EP0_IDLE;
+				if (dev->gadget.speed != USB_SPEED_UNKNOWN
+						&& dev->driver
+						&& dev->driver->resume) {
+					spin_unlock(&dev->lock);
+					dev->driver->resume(&dev->gadget);
+					spin_lock(&dev->lock);
+				}
+			}
+		}
+pm_next:
+		if (stat & INT_USBRESET) {		/* hub reset done */
+			ACK(INT_USBRESET);
+			INFO(dev, "USB reset done, gadget %s\n",
+				dev->driver->driver.name);
+		}
+		// and INT_ERR on some endpoint's crc/bitstuff/... problem
+	}
+
+	/* progress ep0 setup, data, or status stages.
+	 * no transition {EP0_STATUS, EP0_STALL} --> EP0_IDLE; saves irqs
+	 */
+	if (stat & INT_SETUP) {
+		ACK(INT_SETUP);
+		dev->ep[0].irqs++;
+		ep0_setup(dev);
+	}
+        if (stat & INT_STATUSNAK) {
+		ACK(INT_STATUSNAK|INT_ENDPOINT0);
+		if (dev->ep0state == EP0_IN) {
+			ep = &dev->ep[0];
+			ep->irqs++;
+			nuke(ep, 0);
+			writel(~(1<<0), &regs->EOP);
+			dev->ep0state = EP0_STATUS;
+		}
+	}
+        if (stat & INT_ENDPOINT0) {
+		ACK(INT_ENDPOINT0);
+		ep = &dev->ep[0];
+		ep->irqs++;
+		pio_advance(ep);
+        }
+
+	/* dma completion */
+        if (stat & INT_MSTRDEND) {	/* IN */
+		ACK(INT_MSTRDEND);
+		ep = &dev->ep[UDC_MSTRD_ENDPOINT];
+		ep->irqs++;
+		dma_advance(dev, ep);
+        }
+        if (stat & INT_MSTWREND) {	/* OUT */
+		ACK(INT_MSTWREND);
+		ep = &dev->ep[UDC_MSTWR_ENDPOINT];
+		ep->irqs++;
+		dma_advance(dev, ep);
+        }
+        if (stat & INT_MSTWRTMOUT) {	/* OUT */
+		ACK(INT_MSTWRTMOUT);
+		ep = &dev->ep[UDC_MSTWR_ENDPOINT];
+		ep->irqs++;
+		ERROR(dev, "%s write timeout ?\n", ep->ep.name);
+		// reset dma? then dma_advance()
+        }
+
+	/* pio */
+	for (i = 1; i < 4; i++) {
+		u32		tmp = INT_EPxDATASET(i);
+
+		if (!(stat & tmp))
+			continue;
+		ep = &dev->ep[i];
+		pio_advance(ep);
+		if (list_empty (&ep->queue))
+			pio_irq_disable(dev, regs, i);
+		stat &= ~tmp;
+		handled = 1;
+		ep->irqs++;
+	}
+
+	if (rescans--)
+		goto rescan;
+
+done:
+	(void)readl(&regs->int_enable);
+	spin_unlock(&dev->lock);
+	if (stat)
+		DBG(dev, "unhandled irq status: %05x (%05x, %05x)\n", stat,
+				readl(&regs->int_status), dev->int_enable);
+	return IRQ_RETVAL(handled);
+}
+
+#undef ACK
+
+/*-------------------------------------------------------------------------*/
+
+static void gadget_release(struct device *_dev)
+{
+	struct goku_udc	*dev = dev_get_drvdata(_dev);
+
+	kfree(dev);
+}
+
+/* tear down the binding between this driver and the pci device */
+
+static void goku_remove(struct pci_dev *pdev)
+{
+	struct goku_udc		*dev = pci_get_drvdata(pdev);
+
+	DBG(dev, "%s\n", __func__);
+
+	usb_del_gadget_udc(&dev->gadget);
+
+	BUG_ON(dev->driver);
+
+#ifdef CONFIG_USB_GADGET_DEBUG_FILES
+	remove_proc_entry(proc_node_name, NULL);
+#endif
+	if (dev->regs)
+		udc_reset(dev);
+	if (dev->got_irq)
+		free_irq(pdev->irq, dev);
+	if (dev->regs)
+		iounmap(dev->regs);
+	if (dev->got_region)
+		release_mem_region(pci_resource_start (pdev, 0),
+				pci_resource_len (pdev, 0));
+	if (dev->enabled)
+		pci_disable_device(pdev);
+
+	dev->regs = NULL;
+
+	INFO(dev, "unbind\n");
+}
+
+/* wrap this driver around the specified pci device, but
+ * don't respond over USB until a gadget driver binds to us.
+ */
+
+static int goku_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+	struct goku_udc		*dev = NULL;
+	unsigned long		resource, len;
+	void __iomem		*base = NULL;
+	int			retval;
+
+	if (!pdev->irq) {
+		printk(KERN_ERR "Check PCI %s IRQ setup!\n", pci_name(pdev));
+		retval = -ENODEV;
+		goto err;
+	}
+
+	/* alloc, and start init */
+	dev = kzalloc (sizeof *dev, GFP_KERNEL);
+	if (!dev) {
+		retval = -ENOMEM;
+		goto err;
+	}
+
+	pci_set_drvdata(pdev, dev);
+	spin_lock_init(&dev->lock);
+	dev->pdev = pdev;
+	dev->gadget.ops = &goku_ops;
+	dev->gadget.max_speed = USB_SPEED_FULL;
+
+	/* the "gadget" abstracts/virtualizes the controller */
+	dev->gadget.name = driver_name;
+
+	/* now all the pci goodies ... */
+	retval = pci_enable_device(pdev);
+	if (retval < 0) {
+		DBG(dev, "can't enable, %d\n", retval);
+		goto err;
+	}
+	dev->enabled = 1;
+
+	resource = pci_resource_start(pdev, 0);
+	len = pci_resource_len(pdev, 0);
+	if (!request_mem_region(resource, len, driver_name)) {
+		DBG(dev, "controller already in use\n");
+		retval = -EBUSY;
+		goto err;
+	}
+	dev->got_region = 1;
+
+	base = ioremap_nocache(resource, len);
+	if (base == NULL) {
+		DBG(dev, "can't map memory\n");
+		retval = -EFAULT;
+		goto err;
+	}
+	dev->regs = (struct goku_udc_regs __iomem *) base;
+
+	INFO(dev, "%s\n", driver_desc);
+	INFO(dev, "version: " DRIVER_VERSION " %s\n", dmastr());
+	INFO(dev, "irq %d, pci mem %p\n", pdev->irq, base);
+
+	/* init to known state, then setup irqs */
+	udc_reset(dev);
+	udc_reinit (dev);
+	if (request_irq(pdev->irq, goku_irq, IRQF_SHARED,
+			driver_name, dev) != 0) {
+		DBG(dev, "request interrupt %d failed\n", pdev->irq);
+		retval = -EBUSY;
+		goto err;
+	}
+	dev->got_irq = 1;
+	if (use_dma)
+		pci_set_master(pdev);
+
+
+#ifdef CONFIG_USB_GADGET_DEBUG_FILES
+	proc_create_single_data(proc_node_name, 0, NULL, udc_proc_read, dev);
+#endif
+
+	retval = usb_add_gadget_udc_release(&pdev->dev, &dev->gadget,
+			gadget_release);
+	if (retval)
+		goto err;
+
+	return 0;
+
+err:
+	if (dev)
+		goku_remove (pdev);
+	/* gadget_release is not registered yet, kfree explicitly */
+	kfree(dev);
+	return retval;
+}
+
+
+/*-------------------------------------------------------------------------*/
+
+static const struct pci_device_id pci_ids[] = { {
+	.class =	PCI_CLASS_SERIAL_USB_DEVICE,
+	.class_mask =	~0,
+	.vendor =	0x102f,		/* Toshiba */
+	.device =	0x0107,		/* this UDC */
+	.subvendor =	PCI_ANY_ID,
+	.subdevice =	PCI_ANY_ID,
+
+}, { /* end: all zeroes */ }
+};
+MODULE_DEVICE_TABLE (pci, pci_ids);
+
+static struct pci_driver goku_pci_driver = {
+	.name =		(char *) driver_name,
+	.id_table =	pci_ids,
+
+	.probe =	goku_probe,
+	.remove =	goku_remove,
+
+	/* FIXME add power management support */
+};
+
+module_pci_driver(goku_pci_driver);