Skip to content

L
linux
Commit adffcfda authored by David Brownell's avatar David Brownell Committed by Greg Kroah-Hartman
Browse files
Options

[PATCH] USB: ehci update: 3/3, highspeed iso rewrite 

This is an updated version of a patch submitted to me from
Michal Sojka <sojkam1@fel.cvut.cz>, basically providing a
much-needed rewrite of the highspeed ISO support.  I updated
the scheduling and made it a closer match to how OHCI works;
and also tested it a bunch.

So far it seems most of the requests for highspeed ISO support
have been for realtime data collection -- custom apps, nothing
a mainstream kernel would ship with.   The USB Video class is
now defined; highspeed video will also need these updates.

Key changes:

   - Define an "iso_stream" head for iso endpoints.  This acts
     enough like a QH that endpoint_disable() works.  It holds the
     queue of ITDs, and the endpoint's current schedule state.
     And it's easy to find (spinlocked array access), no search.

   - Uses a temporary "itd_sched" while submitting each URB, with
     not-yet-linked ITDs and request-specific metadata.  There's
     a per-stream cache of ITDs, so resubmitting ISO urbs (to
     achieve a "ring" of transfers) is typically cheap.

   - Scheduling for most URBs is almost a NOP:  just a sanity
     check to make sure there's no need to reschedule, and then
     just link into the schedule at the current schedule slot.
     (The previous code was a gross hack that didn't even work
     reasonably with more than two URBs queued.)

   - Is a reasonable model to use with full speed ISO transfers.
     (They need additional TT scheduling hooks, most of which
     are already written but not merged.)

   - Handles several cases the previous code didn't, including
     high bandwidth transfers (loads up to 24 MByte/sec)

   - Has had more testing than the old code, including 20+ hour
     successful IN+OUT runs, more varied transfer intervals and
     maxpacket sizes.  (Using net2280 and a gadgetfs driver.)

So it's worth replacing the existing code with this; there
aren't too many rough edges, and it's much more fixable than
the previous version.


p.s. Many thanks, Michal!
parent 49597169
Show whitespace changes
Inline Side-by-side
Showing with 661 additions and 313 deletions
drivers/usb/host/ehci-dbg.c
View file @ adffcfda
...@@ -140,6 +140,36 @@ dbg_qh (const char *label, struct ehci_hcd *ehci, struct ehci_qh *qh) ...@@ -140,6 +140,36 @@ dbg_qh (const char *label, struct ehci_hcd *ehci, struct ehci_qh *qh)
dbg_qtd ("overlay", ehci, (struct ehci_qtd *) &qh->hw_qtd_next); dbg_qtd ("overlay", ehci, (struct ehci_qtd *) &qh->hw_qtd_next);
} }
static void __attribute__((__unused__))
dbg_itd (const char *label, struct ehci_hcd *ehci, struct ehci_itd *itd)
{
ehci_dbg (ehci, "%s [%d] itd %p, next %08x, urb %p\n",
label, itd->frame, itd, le32_to_cpu(itd->hw_next), itd->urb);
ehci_dbg (ehci,
" trans: %08x %08x %08x %08x %08x %08x %08x %08x\n",
le32_to_cpu(itd->hw_transaction[0]),
le32_to_cpu(itd->hw_transaction[1]),
le32_to_cpu(itd->hw_transaction[2]),
le32_to_cpu(itd->hw_transaction[3]),
le32_to_cpu(itd->hw_transaction[4]),
le32_to_cpu(itd->hw_transaction[5]),
le32_to_cpu(itd->hw_transaction[6]),
le32_to_cpu(itd->hw_transaction[7]));
ehci_dbg (ehci,
" buf: %08x %08x %08x %08x %08x %08x %08x\n",
le32_to_cpu(itd->hw_bufp[0]),
le32_to_cpu(itd->hw_bufp[1]),
le32_to_cpu(itd->hw_bufp[2]),
le32_to_cpu(itd->hw_bufp[3]),
le32_to_cpu(itd->hw_bufp[4]),
le32_to_cpu(itd->hw_bufp[5]),
le32_to_cpu(itd->hw_bufp[6]));
ehci_dbg (ehci, " index: %d %d %d %d %d %d %d %d\n",
itd->index[0], itd->index[1], itd->index[2],
itd->index[3], itd->index[4], itd->index[5],
itd->index[6], itd->index[7]);
}
static int __attribute__((__unused__)) static int __attribute__((__unused__))
dbg_status_buf (char *buf, unsigned len, char *label, u32 status) dbg_status_buf (char *buf, unsigned len, char *label, u32 status)
{ {
......
drivers/usb/host/ehci-hcd.c
View file @ adffcfda
...@@ -67,6 +67,9 @@ ...@@ -67,6 +67,9 @@
* *
* HISTORY: * HISTORY:
* *
* 2003-12-29 Rewritten high speed iso transfer support (by Michal Sojka,
* <sojkam@centrum.cz>, updates by DB).
*
* 2002-11-29 Correct handling for hw async_next register. * 2002-11-29 Correct handling for hw async_next register.
* 2002-08-06 Handling for bulk and interrupt transfers is mostly shared; * 2002-08-06 Handling for bulk and interrupt transfers is mostly shared;
* only scheduling is different, no arbitrary limitations. * only scheduling is different, no arbitrary limitations.
...@@ -90,7 +93,7 @@ ...@@ -90,7 +93,7 @@
* 2001-June Works with usb-storage and NEC EHCI on 2.4 * 2001-June Works with usb-storage and NEC EHCI on 2.4
*/ */
#define DRIVER_VERSION "2003-Jun-13" #define DRIVER_VERSION "2003-Dec-29"
#define DRIVER_AUTHOR "David Brownell" #define DRIVER_AUTHOR "David Brownell"
#define DRIVER_DESC "USB 2.0 'Enhanced' Host Controller (EHCI) Driver" #define DRIVER_DESC "USB 2.0 'Enhanced' Host Controller (EHCI) Driver"
...@@ -901,10 +904,19 @@ ehci_endpoint_disable (struct usb_hcd *hcd, struct hcd_dev *dev, int ep) ...@@ -901,10 +904,19 @@ ehci_endpoint_disable (struct usb_hcd *hcd, struct hcd_dev *dev, int ep)
if (!qh) if (!qh)
goto done; goto done;
/* endpoints can be iso streams. for now, we don't
* accelerate iso completions ... so spin a while.
*/
if (qh->hw_info1 == 0) {
ehci_vdbg (ehci, "iso delay\n");
goto idle_timeout;
}
if (!HCD_IS_RUNNING (ehci->hcd.state)) if (!HCD_IS_RUNNING (ehci->hcd.state))
qh->qh_state = QH_STATE_IDLE; qh->qh_state = QH_STATE_IDLE;
switch (qh->qh_state) { switch (qh->qh_state) {
case QH_STATE_UNLINK: /* wait for hw to finish? */ case QH_STATE_UNLINK: /* wait for hw to finish? */
idle_timeout:
spin_unlock_irqrestore (&ehci->lock, flags); spin_unlock_irqrestore (&ehci->lock, flags);
set_current_state (TASK_UNINTERRUPTIBLE); set_current_state (TASK_UNINTERRUPTIBLE);
schedule_timeout (1); schedule_timeout (1);
......
drivers/usb/host/ehci-sched.c
View file @ adffcfda
/* /*
* Copyright (c) 2001-2002 by David Brownell * Copyright (c) 2001-2003 by David Brownell
* Copyright (c) 2003 Michal Sojka, for high-speed iso transfers
* *
* This program is free software; you can redistribute it and/or modify it * This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the * under the terms of the GNU General Public License as published by the
...@@ -27,10 +28,10 @@ ...@@ -27,10 +28,10 @@
* Note that for interrupt transfers, the QH/QTD manipulation is shared * Note that for interrupt transfers, the QH/QTD manipulation is shared
* with the "asynchronous" transaction support (control/bulk transfers). * with the "asynchronous" transaction support (control/bulk transfers).
* The only real difference is in how interrupt transfers are scheduled. * The only real difference is in how interrupt transfers are scheduled.
* We get some funky API restrictions from the current URB model, which *
* works notably better for reading transfers than for writing. (And * For ISO, we make an "iso_stream" head to serve the same role as a QH.
* which accordingly needs to change before it'll work inside devices, * It keeps track of every ITD (or SITD) that's linked, and holds enough
* or with "USB On The Go" additions to USB 2.0 ...) * pre-calculated schedule data to make appending to the queue be quick.
*/ */
static int ehci_get_frame (struct usb_hcd *hcd); static int ehci_get_frame (struct usb_hcd *hcd);
...@@ -126,9 +127,7 @@ periodic_usecs (struct ehci_hcd *ehci, unsigned frame, unsigned uframe) ...@@ -126,9 +127,7 @@ periodic_usecs (struct ehci_hcd *ehci, unsigned frame, unsigned uframe)
q = &q->fstn->fstn_next; q = &q->fstn->fstn_next;
break; break;
case Q_TYPE_ITD: case Q_TYPE_ITD:
/* NOTE the "one uframe per itd" policy */ usecs += q->itd->usecs [uframe];
if (q->itd->hw_transaction [uframe] != 0)
usecs += q->itd->usecs;
q = &q->itd->itd_next; q = &q->itd->itd_next;
break; break;
#ifdef have_split_iso #ifdef have_split_iso
...@@ -520,348 +519,563 @@ intr_complete ( ...@@ -520,348 +519,563 @@ intr_complete (
/*-------------------------------------------------------------------------*/ /*-------------------------------------------------------------------------*/
static void static inline struct ehci_iso_stream *
itd_free_list (struct ehci_hcd *ehci, struct urb *urb) iso_stream_alloc (int mem_flags)
{ {
struct ehci_itd *first_itd = urb->hcpriv; struct ehci_iso_stream *stream;
while (!list_empty (&first_itd->itd_list)) { stream = kmalloc(sizeof *stream, mem_flags);
struct ehci_itd *itd; if (likely (stream != 0)) {
memset (stream, 0, sizeof(*stream));
itd = list_entry ( INIT_LIST_HEAD(&stream->itd_list);
first_itd->itd_list.next, INIT_LIST_HEAD(&stream->free_itd_list);
struct ehci_itd, itd_list); stream->next_uframe = -1;
list_del (&itd->itd_list); stream->refcount = 1;
pci_pool_free (ehci->itd_pool, itd, itd->itd_dma);
} }
pci_pool_free (ehci->itd_pool, first_itd, first_itd->itd_dma); return stream;
urb->hcpriv = 0;
} }
static int static inline void
itd_fill ( iso_stream_init (
struct ehci_hcd *ehci, struct ehci_iso_stream *stream,
struct ehci_itd *itd, struct usb_device *dev,
struct urb *urb, int pipe,
unsigned index, // urb->iso_frame_desc [index] unsigned interval
dma_addr_t dma // mapped transfer buffer )
) { {
u64 temp;
u32 buf1; u32 buf1;
unsigned i, epnum, maxp, multi; unsigned epnum, maxp, multi;
unsigned length;
int is_input; int is_input;
long bandwidth;
itd->hw_next = EHCI_LIST_END;
itd->urb = urb;
itd->index = index;
/* tell itd about its transfer buffer, max 2 pages */
length = urb->iso_frame_desc [index].length;
dma += urb->iso_frame_desc [index].offset;
temp = dma & ~0x0fff;
for (i = 0; i < 2; i++) {
itd->hw_bufp [i] = cpu_to_le32 ((u32) temp);
itd->hw_bufp_hi [i] = cpu_to_le32 ((u32)(temp >> 32));
temp += 0x1000;
}
itd->buf_dma = dma;
/* /*
* this might be a "high bandwidth" highspeed endpoint, * this might be a "high bandwidth" highspeed endpoint,
* as encoded in the ep descriptor's maxpacket field * as encoded in the ep descriptor's wMaxPacket field
*/ */
epnum = usb_pipeendpoint (urb->pipe); epnum = usb_pipeendpoint (pipe);
is_input = usb_pipein (urb->pipe); is_input = usb_pipein (pipe) ? USB_DIR_IN : 0;
if (is_input) { if (is_input) {
maxp = urb->dev->epmaxpacketin [epnum]; maxp = dev->epmaxpacketin [epnum];
buf1 = (1 << 11); buf1 = (1 << 11);
} else { } else {
maxp = urb->dev->epmaxpacketout [epnum]; maxp = dev->epmaxpacketout [epnum];
buf1 = 0; buf1 = 0;
} }
buf1 |= (maxp & 0x03ff);
multi = 1; multi = hb_mult(maxp);
multi += (maxp >> 11) & 0x03; maxp = max_packet(maxp);
maxp &= 0x03ff; buf1 |= maxp;
maxp *= multi; maxp *= multi;
/* transfer can't fit in any uframe? */ stream->dev = (struct hcd_dev *)dev->hcpriv;
if (length < 0 || maxp < length) {
dbg ("BAD iso packet: %d bytes, max %d, urb %p [%d] (of %d)",
length, maxp, urb, index,
urb->iso_frame_desc [index].length);
return -ENOSPC;
}
itd->usecs = usb_calc_bus_time (USB_SPEED_HIGH, is_input, 1, length);
/* "plus" info in low order bits of buffer pointers */ stream->bEndpointAddress = is_input | epnum;
itd->hw_bufp [0] |= cpu_to_le32 ((epnum << 8) | urb->dev->devnum); stream->interval = interval;
itd->hw_bufp [1] |= cpu_to_le32 (buf1); stream->maxp = maxp;
itd->hw_bufp [2] |= cpu_to_le32 (multi);
/* figure hw_transaction[] value (it's scheduled later) */ stream->buf0 = cpu_to_le32 ((epnum << 8) | dev->devnum);
itd->transaction = EHCI_ISOC_ACTIVE; stream->buf1 = cpu_to_le32 (buf1);
itd->transaction |= dma & 0x0fff; /* offset; buffer=0 */ stream->buf2 = cpu_to_le32 (multi);
if ((index + 1) == urb->number_of_packets)
itd->transaction |= EHCI_ITD_IOC; /* end-of-urb irq */
itd->transaction |= length << 16;
cpu_to_le32s (&itd->transaction);
return 0; /* usbfs wants to report the average usecs per frame tied up
* when transfers on this endpoint are scheduled ...
*/
stream->usecs = HS_USECS_ISO (maxp);
bandwidth = stream->usecs * 8;
bandwidth /= 1 << (interval - 1);
stream->bandwidth = bandwidth;
} }
static int static void
itd_urb_transaction ( iso_stream_put(struct ehci_hcd *ehci, struct ehci_iso_stream *stream)
struct ehci_hcd *ehci, {
struct urb *urb, stream->refcount--;
int mem_flags
) {
int frame_index;
struct ehci_itd *first_itd, *itd;
int status;
dma_addr_t itd_dma;
/* allocate/init ITDs */ /* free whenever just a dev->ep reference remains.
for (frame_index = 0, first_itd = 0; * not like a QH -- no persistent state (toggle, halt)
frame_index < urb->number_of_packets; */
frame_index++) { if (stream->refcount == 1) {
itd = pci_pool_alloc (ehci->itd_pool, mem_flags, &itd_dma); int is_in;
if (!itd) {
status = -ENOMEM; // BUG_ON (!list_empty(&stream->itd_list));
goto fail;
while (!list_empty (&stream->free_itd_list)) {
struct ehci_itd *itd;
itd = list_entry (stream->free_itd_list.next,
struct ehci_itd, itd_list);
list_del (&itd->itd_list);
pci_pool_free (ehci->itd_pool, itd, itd->itd_dma);
} }
memset (itd, 0, sizeof *itd);
itd->itd_dma = itd_dma;
status = itd_fill (ehci, itd, urb, frame_index, is_in = (stream->bEndpointAddress & USB_DIR_IN) ? 0x10 : 0;
urb->transfer_dma); stream->bEndpointAddress &= 0x0f;
if (status != 0) stream->dev->ep [is_in + stream->bEndpointAddress] = 0;
goto fail;
if (stream->rescheduled) {
ehci_info (ehci, "ep%d%s-iso rescheduled "
"%lu times in %lu seconds\n",
stream->bEndpointAddress, is_in ? "in" : "out",
stream->rescheduled,
((jiffies - stream->start)/HZ)
);
}
if (first_itd) kfree(stream);
list_add_tail (&itd->itd_list, }
&first_itd->itd_list); }
else {
INIT_LIST_HEAD (&itd->itd_list); static inline struct ehci_iso_stream *
urb->hcpriv = first_itd = itd; iso_stream_get (struct ehci_iso_stream *stream)
{
if (likely (stream != 0))
stream->refcount++;
return stream;
}
static struct ehci_iso_stream *
iso_stream_find (struct ehci_hcd *ehci, struct urb *urb)
{
unsigned epnum;
struct hcd_dev *dev;
struct ehci_iso_stream *stream;
unsigned long flags;
epnum = usb_pipeendpoint (urb->pipe);
if (usb_pipein(urb->pipe))
epnum += 0x10;
spin_lock_irqsave (&ehci->lock, flags);
dev = (struct hcd_dev *)urb->dev->hcpriv;
stream = dev->ep [epnum];
if (unlikely (stream == 0)) {
stream = iso_stream_alloc(GFP_ATOMIC);
if (likely (stream != 0)) {
/* dev->ep owns the initial refcount */
dev->ep[epnum] = stream;
iso_stream_init(stream, urb->dev, urb->pipe,
urb->interval);
} }
/* if dev->ep [epnum] is a QH, info1.maxpacket is nonzero */
} else if (unlikely (stream->hw_info1 != 0)) {
ehci_dbg (ehci, "dev %s ep%d%s, not iso??\n",
urb->dev->devpath, epnum & 0x0f,
(epnum & 0x10) ? "in" : "out");
stream = 0;
} }
urb->error_count = 0;
return 0;
fail: /* caller guarantees an eventual matching iso_stream_put */
if (urb->hcpriv) stream = iso_stream_get (stream);
itd_free_list (ehci, urb);
return status; spin_unlock_irqrestore (&ehci->lock, flags);
return stream;
} }
/*-------------------------------------------------------------------------*/ /*-------------------------------------------------------------------------*/
static inline void static inline struct ehci_itd_sched *
itd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_itd *itd) itd_sched_alloc (unsigned packets, int mem_flags)
{ {
/* always prepend ITD/SITD ... only QH tree is order-sensitive */ struct ehci_itd_sched *itd_sched;
itd->itd_next = ehci->pshadow [frame]; int size = sizeof *itd_sched;
itd->hw_next = ehci->periodic [frame];
ehci->pshadow [frame].itd = itd; size += packets * sizeof (struct ehci_iso_uframe);
ehci->periodic [frame] = cpu_to_le32 (itd->itd_dma) | Q_TYPE_ITD; itd_sched = kmalloc (size, mem_flags);
if (likely (itd_sched != 0)) {
memset(itd_sched, 0, size);
INIT_LIST_HEAD (&itd_sched->itd_list);
}
return itd_sched;
} }
/* static int
* return zero on success, else -errno itd_sched_init (
* - start holds first uframe to start scheduling into struct ehci_itd_sched *itd_sched,
* - max is the first uframe it's NOT (!) OK to start scheduling into struct ehci_iso_stream *stream,
* math to be done modulo "mod" (ehci->periodic_size << 3) struct urb *urb
)
{
unsigned i;
dma_addr_t dma = urb->transfer_dma;
/* how many uframes are needed for these transfers */
itd_sched->span = urb->number_of_packets * stream->interval;
/* figure out per-uframe itd fields that we'll need later
* when we fit new itds into the schedule.
*/ */
static int get_iso_range ( for (i = 0; i < urb->number_of_packets; i++) {
struct ehci_hcd *ehci, struct ehci_iso_uframe *uframe = &itd_sched->packet [i];
struct urb *urb, unsigned length;
unsigned *start, dma_addr_t buf;
unsigned *max, u32 trans;
unsigned mod
) { length = urb->iso_frame_desc [i].length;
struct list_head *lh; buf = dma + urb->iso_frame_desc [i].offset;
struct hcd_dev *dev = urb->dev->hcpriv;
int last = -1; trans = EHCI_ISOC_ACTIVE;
unsigned now, span, end; trans |= buf & 0x0fff;
if (unlikely ((i + 1) == urb->number_of_packets))
trans |= EHCI_ITD_IOC;
trans |= length << 16;
uframe->transaction = cpu_to_le32 (trans);
/* might need to cross a buffer page within a td */
uframe->bufp = (buf & ~(u64)0x0fff);
buf += length;
if (unlikely ((uframe->bufp != (buf & ~(u64)0x0fff))))
uframe->cross = 1;
}
return 0;
}
span = urb->interval * urb->number_of_packets; static void
itd_sched_free (
struct ehci_iso_stream *stream,
struct ehci_itd_sched *itd_sched
)
{
list_splice (&itd_sched->itd_list, &stream->free_itd_list);
kfree (itd_sched);
}
/* first see if we know when the next transfer SHOULD happen */ static int
list_for_each (lh, &dev->urb_list) { itd_urb_transaction (
struct urb *u; struct ehci_iso_stream *stream,
struct ehci_hcd *ehci,
struct urb *urb,
int mem_flags
)
{
struct ehci_itd *itd; struct ehci_itd *itd;
unsigned s; int status;
dma_addr_t itd_dma;
int i;
unsigned num_itds;
struct ehci_itd_sched *itd_sched;
u = list_entry (lh, struct urb, urb_list); itd_sched = itd_sched_alloc (urb->number_of_packets, mem_flags);
if (u == urb || u->pipe != urb->pipe) if (unlikely (itd_sched == 0))
continue; return -ENOMEM;
if (u->interval != urb->interval) { /* must not change! */
dbg ("urb %p interval %d ... != %p interval %d", status = itd_sched_init (itd_sched, stream, urb);
u, u->interval, urb, urb->interval); if (unlikely (status != 0)) {
return -EINVAL; itd_sched_free (stream, itd_sched);
return status;
} }
/* URB for this endpoint... covers through when? */ if (urb->interval < 8)
itd = urb->hcpriv; num_itds = 1 + (itd_sched->span + 7) / 8;
s = itd->uframe + u->interval * u->number_of_packets; else
if (last < 0) num_itds = urb->number_of_packets;
last = s;
else { /* allocate/init ITDs */
/* for (i = 0; i < num_itds; i++) {
* So far we can only queue two ISO URBs...
* /* free_itd_list.next might be cache-hot ... but maybe
* FIXME do interval math, figure out whether * the HC caches it too. avoid that issue for now.
* this URB is "before" or not ... also, handle
* the case where the URB might have completed,
* but hasn't yet been processed.
*/ */
dbg ("NYET: queue >2 URBs per ISO endpoint");
return -EDOM;
}
}
/* calculate the legal range [start,max) */ /* prefer previously-allocated itds */
now = readl (&ehci->regs->frame_index) + 1; /* next uframe */ if (likely (!list_empty(&stream->free_itd_list))) {
if (!ehci->periodic_sched) itd = list_entry (stream->free_itd_list.prev,
now += 8; /* startup delay */ struct ehci_itd, itd_list);
now %= mod; list_del (&itd->itd_list);
end = now + mod; itd_dma = itd->itd_dma;
if (last < 0) { } else
*start = now + ehci->i_thresh + /* paranoia */ 1; itd = pci_pool_alloc (ehci->itd_pool, mem_flags,
*max = end - span; &itd_dma);
if (*max < *start + 1)
*max = *start + 1;
} else {
*start = last % mod;
*max = (last + 1) % mod;
}
/* explicit start frame? */ if (unlikely (0 == itd)) {
if (!(urb->transfer_flags & URB_ISO_ASAP)) { itd_sched_free (stream, itd_sched);
unsigned temp; return -ENOMEM;
}
/* sanity check: must be in range */ memset (itd, 0, sizeof *itd);
urb->start_frame %= ehci->periodic_size; itd->itd_dma = itd_dma;
temp = urb->start_frame << 3; list_add (&itd->itd_list, &itd_sched->itd_list);
if (temp < *start)
temp += mod;
if (temp > *max)
return -EDOM;
/* use that explicit start frame */
*start = urb->start_frame << 3;
temp += 8;
if (temp < *max)
*max = temp;
} }
// FIXME minimize wraparound to "now" ... insist max+span /* temporarily store schedule info in hcpriv */
// (and start+span) remains a few frames short of "end" urb->hcpriv = itd_sched;
urb->error_count = 0;
*max %= ehci->periodic_size;
if ((*start + span) < end)
return 0; return 0;
return -EFBIG;
} }
/*
* This scheduler plans almost as far into the future as it has actual
* periodic schedule slots. (Affected by TUNE_FLS, which defaults to
* "as small as possible" to be cache-friendlier.) That limits the size
* transfers you can stream reliably; avoid more than 64 msec per urb.
* Also avoid queue depths of less than the system's worst irq latency.
*/
#define SCHEDULE_SLOP 10 /* frames */
static int static int
itd_schedule (struct ehci_hcd *ehci, struct urb *urb) itd_stream_schedule (
struct ehci_hcd *ehci,
struct urb *urb,
struct ehci_iso_stream *stream
)
{ {
unsigned start, max, i; u32 now, start, end, max;
int status; int status;
unsigned mod = ehci->periodic_size << 3; unsigned mod = ehci->periodic_size << 3;
struct ehci_itd_sched *itd_sched = urb->hcpriv;
for (i = 0; i < urb->number_of_packets; i++) { if (unlikely (itd_sched->span > (mod - 8 * SCHEDULE_SLOP))) {
urb->iso_frame_desc [i].status = -EINPROGRESS; ehci_dbg (ehci, "iso request %p too long\n", urb);
urb->iso_frame_desc [i].actual_length = 0; status = -EFBIG;
goto fail;
} }
if ((status = get_iso_range (ehci, urb, &start, &max, mod)) != 0) now = readl (&ehci->regs->frame_index) % mod;
return status;
/* when's the last uframe this urb could start? */
max = now + mod;
max -= itd_sched->span;
max -= 8 * SCHEDULE_SLOP;
/* typical case: reuse current schedule. stream is still active,
* and no gaps from host falling behind (irq delays etc)
*/
if (likely (!list_empty (&stream->itd_list))) {
start = stream->next_uframe;
if (start < now)
start += mod;
if (likely (start < max))
goto ready;
/* two cases:
* (a) we missed some uframes ... can reschedule
* (b) trying to overcommit the schedule
* FIXME (b) should be a hard failure
*/
}
/* need to schedule; when's the next (u)frame we could start?
* this is bigger than ehci->i_thresh allows; scheduling itself
* isn't free, the slop should handle reasonably slow cpus. it
* can also help high bandwidth if the dma and irq loads don't
* jump until after the queue is primed.
*/
start = SCHEDULE_SLOP * 8 + (now & ~0x07);
end = start;
ehci_vdbg (ehci, "%s schedule from %d (%d..%d), was %d\n",
__FUNCTION__, now, start, max,
stream->next_uframe);
/* NOTE: assumes URB_ISO_ASAP, to limit complexity/bugs */
if (likely (max > (start + urb->interval)))
max = start + urb->interval;
/* hack: account for itds already scheduled to this endpoint */
if (unlikely (list_empty (&stream->itd_list)))
end = max;
/* within [start..max] find a uframe slot with enough bandwidth */
end %= mod;
do { do {
unsigned uframe; unsigned uframe;
unsigned usecs; int enough_space = 1;
struct ehci_itd *itd;
/* check schedule: enough space? */ /* check schedule: enough space? */
itd = urb->hcpriv;
uframe = start; uframe = start;
for (i = 0, uframe = start; do {
i < urb->number_of_packets;
i++, uframe += urb->interval) {
uframe %= mod; uframe %= mod;
/* can't commit more than 80% periodic == 100 usec */ /* can't commit more than 80% periodic == 100 usec */
if (periodic_usecs (ehci, uframe >> 3, uframe & 0x7) if (periodic_usecs (ehci, uframe >> 3, uframe & 0x7)
> (100 - itd->usecs)) { > (100 - stream->usecs)) {
itd = 0; enough_space = 0;
break; break;
} }
itd = list_entry (itd->itd_list.next,
struct ehci_itd, itd_list); /* we know urb->interval is 2^N uframes */
uframe += urb->interval;
} while (uframe != end);
/* (re)schedule it here if there's enough bandwidth */
if (enough_space) {
start %= mod;
if (unlikely (!list_empty (&stream->itd_list))) {
/* host fell behind ... maybe irq latencies
* delayed this request queue for too long.
*/
stream->rescheduled++;
dev_dbg (&urb->dev->dev,
"iso%d%s %d.%d skip %d.%d\n",
stream->bEndpointAddress & 0x0f,
(stream->bEndpointAddress & USB_DIR_IN)
? "in" : "out",
stream->next_uframe >> 3,
stream->next_uframe & 0x7,
start >> 3, start & 0x7);
}
stream->next_uframe = start;
goto ready;
} }
if (!itd)
continue;
/* that's where we'll schedule this! */ } while (++start < max);
itd = urb->hcpriv;
urb->start_frame = start >> 3; /* no room in the schedule */
vdbg ("ISO urb %p (%d packets period %d) starting %d.%d", ehci_dbg (ehci, "iso %ssched full %p (now %d end %d max %d)\n",
urb, urb->number_of_packets, urb->interval, list_empty (&stream->itd_list) ? "" : "re",
urb->start_frame, start & 0x7); urb, now, end, max);
for (i = 0, uframe = start, usecs = 0; status = -ENOSPC;
i < urb->number_of_packets;
i++, uframe += urb->interval) { fail:
uframe %= mod; itd_sched_free (stream, itd_sched);
urb->hcpriv = 0;
return status;
ready:
urb->start_frame = stream->next_uframe;
return 0;
}
/*-------------------------------------------------------------------------*/
static inline void
itd_init (struct ehci_iso_stream *stream, struct ehci_itd *itd)
{
int i;
itd->hw_next = EHCI_LIST_END;
itd->hw_bufp [0] = stream->buf0;
itd->hw_bufp [1] = stream->buf1;
itd->hw_bufp [2] = stream->buf2;
itd->uframe = uframe; for (i = 0; i < 8; i++)
itd->hw_transaction [uframe & 0x07] = itd->transaction; itd->index[i] = -1;
itd_link (ehci, (uframe >> 3) % ehci->periodic_size,
itd); /* All other fields are filled when scheduling */
}
static inline void
itd_patch (
struct ehci_itd *itd,
struct ehci_itd_sched *itd_sched,
unsigned index,
u16 uframe,
int first
)
{
struct ehci_iso_uframe *uf = &itd_sched->packet [index];
unsigned pg = itd->pg;
// BUG_ON (pg == 6 && uf->cross);
uframe &= 0x07;
itd->index [uframe] = index;
itd->hw_transaction [uframe] = uf->transaction;
itd->hw_transaction [uframe] |= cpu_to_le32 (pg << 12);
itd->hw_bufp [pg] |= cpu_to_le32 (uf->bufp & ~(u32)0);
itd->hw_bufp_hi [pg] |= cpu_to_le32 ((u32)(uf->bufp >> 32));
/* iso_frame_desc[].offset must be strictly increasing */
if (unlikely (!first && uf->cross)) {
u64 bufp = uf->bufp + 4096;
itd->pg = ++pg;
itd->hw_bufp [pg] |= cpu_to_le32 (bufp & ~(u32)0);
itd->hw_bufp_hi [pg] |= cpu_to_le32 ((u32)(bufp >> 32));
}
}
static inline void
itd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_itd *itd)
{
/* always prepend ITD/SITD ... only QH tree is order-sensitive */
itd->itd_next = ehci->pshadow [frame];
itd->hw_next = ehci->periodic [frame];
ehci->pshadow [frame].itd = itd;
itd->frame = frame;
wmb (); wmb ();
usecs += itd->usecs; ehci->periodic [frame] = cpu_to_le32 (itd->itd_dma) | Q_TYPE_ITD;
}
itd = list_entry (itd->itd_list.next, /* fit urb's itds into the selected schedule slot; activate as needed */
static int
itd_link_urb (
struct ehci_hcd *ehci,
struct urb *urb,
unsigned mod,
struct ehci_iso_stream *stream
)
{
int packet, first = 1;
unsigned next_uframe, uframe, frame;
struct ehci_itd_sched *itd_sched = urb->hcpriv;
struct ehci_itd *itd;
next_uframe = stream->next_uframe % mod;
if (unlikely (list_empty(&stream->itd_list))) {
hcd_to_bus (&ehci->hcd)->bandwidth_allocated
+= stream->bandwidth;
ehci_vdbg (ehci,
"schedule devp %s ep%d%s-iso period %d start %d.%d\n",
urb->dev->devpath, stream->bEndpointAddress & 0x0f,
(stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
urb->interval,
next_uframe >> 3, next_uframe & 0x7);
stream->start = jiffies;
}
hcd_to_bus (&ehci->hcd)->bandwidth_isoc_reqs++;
/* fill iTDs uframe by uframe */
for (packet = 0, itd = 0; packet < urb->number_of_packets; ) {
if (itd == 0) {
/* ASSERT: we have all necessary itds */
// BUG_ON (list_empty (&itd_sched->itd_list));
/* ASSERT: no itds for this endpoint in this uframe */
itd = list_entry (itd_sched->itd_list.next,
struct ehci_itd, itd_list); struct ehci_itd, itd_list);
list_move_tail (&itd->itd_list, &stream->itd_list);
itd->stream = iso_stream_get (stream);
itd->urb = usb_get_urb (urb);
first = 1;
itd_init (stream, itd);
} }
/* update bandwidth utilization records (for usbfs) uframe = next_uframe & 0x07;
* frame = next_uframe >> 3;
* FIXME This claims each URB queued to an endpoint, as if
* transfers were concurrent, not sequential. So bandwidth
* typically gets double-billed ... comes from tying it to
* URBs rather than endpoints in the schedule. Luckily we
* don't use this usbfs data for serious decision making.
*/
usecs /= urb->number_of_packets;
usecs /= urb->interval;
usecs >>= 3;
if (usecs < 1)
usecs = 1;
usb_claim_bandwidth (urb->dev, urb, usecs, 1);
/* maybe enable periodic schedule processing */ itd->usecs [uframe] = stream->usecs;
if (!ehci->periodic_sched++) { itd_patch (itd, itd_sched, packet, uframe, first);
if ((status = enable_periodic (ehci)) != 0) { first = 0;
// FIXME deschedule right away
err ("itd_schedule, enable = %d", status); next_uframe += stream->interval;
next_uframe %= mod;
packet++;
/* link completed itds into the schedule */
if (((next_uframe >> 3) != frame)
|| packet == urb->number_of_packets) {
itd_link (ehci, frame % ehci->periodic_size, itd);
itd = 0;
} }
} }
stream->next_uframe = next_uframe;
return 0; /* don't need that schedule data any more */
itd_sched_free (stream, itd_sched);
} while ((start = ++start % mod) != max); urb->hcpriv = 0;
/* no room in the schedule */ if (unlikely (!ehci->periodic_sched++))
dbg ("urb %p, CAN'T SCHEDULE", urb); return enable_periodic (ehci);
return -ENOSPC; return 0;
} }
/*-------------------------------------------------------------------------*/
#define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR) #define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR)
static unsigned static unsigned
...@@ -875,12 +1089,14 @@ itd_complete ( ...@@ -875,12 +1089,14 @@ itd_complete (
u32 t; u32 t;
unsigned uframe; unsigned uframe;
int urb_index = -1; int urb_index = -1;
struct ehci_iso_stream *stream = itd->stream;
struct usb_device *dev;
/* for each uframe with a packet */ /* for each uframe with a packet */
for (uframe = 0; uframe < 8; uframe++) { for (uframe = 0; uframe < 8; uframe++) {
if (itd->hw_transaction [uframe] == 0) if (likely (itd->index[uframe] == -1))
continue; continue;
urb_index = itd->index; urb_index = itd->index[uframe];
desc = &urb->iso_frame_desc [urb_index]; desc = &urb->iso_frame_desc [urb_index];
t = le32_to_cpup (&itd->hw_transaction [uframe]); t = le32_to_cpup (&itd->hw_transaction [uframe]);
...@@ -907,36 +1123,42 @@ itd_complete ( ...@@ -907,36 +1123,42 @@ itd_complete (
} }
} }
vdbg ("itd %p urb %p packet %d/%d trans %x status %d len %d", usb_put_urb (urb);
itd, urb, itd->index + 1, urb->number_of_packets, itd->urb = 0;
t, desc->status, desc->actual_length); itd->stream = 0;
list_move (&itd->itd_list, &stream->free_itd_list);
iso_stream_put (ehci, stream);
/* handle completion now? */ /* handle completion now? */
if (likely ((urb_index + 1) != urb->number_of_packets)) if (likely ((urb_index + 1) != urb->number_of_packets))
return 0; return 0;
/* /* ASSERT: it's really the last itd for this urb
* Always give the urb back to the driver ... expect it to submit list_for_each_entry (itd, &stream->itd_list, itd_list)
* a new urb (or resubmit this), and to have another already queued BUG_ON (itd->urb == urb);
* when un-interrupted transfers are needed.
*
* NOTE that for now we don't accelerate ISO unlinks; they just
* happen according to the current schedule. Means a delay of
* up to about a second (max).
*/ */
itd_free_list (ehci, urb);
if (urb->status == -EINPROGRESS)
urb->status = 0;
/* complete() can reenter this HCD */ /* give urb back to the driver ... can be out-of-order */
spin_unlock (&ehci->lock); dev = usb_get_dev (urb->dev);
usb_hcd_giveback_urb (&ehci->hcd, urb, regs); ehci_urb_done (ehci, urb, regs);
spin_lock (&ehci->lock); urb = 0;
/* defer stopping schedule; completion can submit */ /* defer stopping schedule; completion can submit */
ehci->periodic_sched--; ehci->periodic_sched--;
if (!ehci->periodic_sched) if (unlikely (!ehci->periodic_sched))
(void) disable_periodic (ehci); (void) disable_periodic (ehci);
hcd_to_bus (&ehci->hcd)->bandwidth_isoc_reqs--;
if (unlikely (list_empty (&stream->itd_list))) {
hcd_to_bus (&ehci->hcd)->bandwidth_allocated
-= stream->bandwidth;
ehci_vdbg (ehci,
"deschedule devp %s ep%d%s-iso\n",
dev->devpath, stream->bEndpointAddress & 0x0f,
(stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");
}
iso_stream_put (ehci, stream);
usb_put_dev (dev);
return 1; return 1;
} }
...@@ -945,23 +1167,50 @@ itd_complete ( ...@@ -945,23 +1167,50 @@ itd_complete (
static int itd_submit (struct ehci_hcd *ehci, struct urb *urb, int mem_flags) static int itd_submit (struct ehci_hcd *ehci, struct urb *urb, int mem_flags)
{ {
int status; int status = -EINVAL;
unsigned long flags; unsigned long flags;
struct ehci_iso_stream *stream;
dbg ("itd_submit urb %p", urb); /* Get iso_stream head */
stream = iso_stream_find (ehci, urb);
if (unlikely (stream == 0)) {
ehci_dbg (ehci, "can't get iso stream\n");
return -ENOMEM;
}
if (unlikely (urb->interval != stream->interval)) {
ehci_dbg (ehci, "can't change iso interval %d --> %d\n",
stream->interval, urb->interval);
goto done;
}
#ifdef EHCI_URB_TRACE
ehci_dbg (ehci,
"%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n",
__FUNCTION__, urb->dev->devpath, urb,
usb_pipeendpoint (urb->pipe),
usb_pipein (urb->pipe) ? "in" : "out",
urb->transfer_buffer_length,
urb->number_of_packets, urb->interval,
stream);
#endif
/* allocate ITDs w/o locking anything */ /* allocate ITDs w/o locking anything */
status = itd_urb_transaction (ehci, urb, mem_flags); status = itd_urb_transaction (stream, ehci, urb, mem_flags);
if (status < 0) if (unlikely (status < 0)) {
return status; ehci_dbg (ehci, "can't init itds\n");
goto done;
}
/* schedule ... need to lock */ /* schedule ... need to lock */
spin_lock_irqsave (&ehci->lock, flags); spin_lock_irqsave (&ehci->lock, flags);
status = itd_schedule (ehci, urb); status = itd_stream_schedule (ehci, urb, stream);
if (likely (status == 0))
itd_link_urb (ehci, urb, ehci->periodic_size << 3, stream);
spin_unlock_irqrestore (&ehci->lock, flags); spin_unlock_irqrestore (&ehci->lock, flags);
if (status < 0)
itd_free_list (ehci, urb);
done:
if (unlikely (status < 0))
iso_stream_put (ehci, stream);
return status; return status;
} }
......
drivers/usb/host/ehci.h
View file @ adffcfda
...@@ -386,6 +386,63 @@ struct ehci_qh { ...@@ -386,6 +386,63 @@ struct ehci_qh {
/*-------------------------------------------------------------------------*/ /*-------------------------------------------------------------------------*/
/* description of one iso highspeed transaction (up to 3 KB data) */
struct ehci_iso_uframe {
/* These will be copied to iTD when scheduling */
u64 bufp; /* itd->hw_bufp{,_hi}[pg] |= */
u32 transaction; /* itd->hw_transaction[i] |= */
u8 cross; /* buf crosses pages */
};
/* temporary schedule data for highspeed packets from iso urbs
* each packet is one uframe's usb transactions, in some itd,
* beginning at stream->next_uframe
*/
struct ehci_itd_sched {
struct list_head itd_list;
unsigned span;
struct ehci_iso_uframe packet [0];
};
/*
* ehci_iso_stream - groups all (s)itds for this endpoint.
* acts like a qh would, if EHCI had them for ISO.
*/
struct ehci_iso_stream {
/* first two fields match QH, but info1 == 0 */
u32 hw_next;
u32 hw_info1;
u32 refcount;
u8 bEndpointAddress;
struct list_head itd_list; /* queued itds */
struct list_head free_itd_list; /* list of unused itds */
struct hcd_dev *dev;
/* output of (re)scheduling */
unsigned long start; /* jiffies */
unsigned long rescheduled;
int next_uframe;
/* the rest is derived from the endpoint descriptor,
* trusting urb->interval == (1 << (epdesc->bInterval - 1)),
* including the extra info for hw_bufp[0..2]
*/
u8 interval;
u8 usecs;
u16 maxp;
unsigned bandwidth;
/* This is used to initialize iTD's hw_bufp fields */
u32 buf0;
u32 buf1;
u32 buf2;
/* ... sITD won't use buf[012], and needs TT access ... */
};
/*-------------------------------------------------------------------------*/
/* /*
* EHCI Specification 0.95 Section 3.3 * EHCI Specification 0.95 Section 3.3
* Fig 3-4 "Isochronous Transaction Descriptor (iTD)" * Fig 3-4 "Isochronous Transaction Descriptor (iTD)"
...@@ -413,14 +470,14 @@ struct ehci_itd { ...@@ -413,14 +470,14 @@ struct ehci_itd {
union ehci_shadow itd_next; /* ptr to periodic q entry */ union ehci_shadow itd_next; /* ptr to periodic q entry */
struct urb *urb; struct urb *urb;
struct list_head itd_list; /* list of urb frames' itds */ struct ehci_iso_stream *stream; /* endpoint's queue */
dma_addr_t buf_dma; /* frame's buffer address */ struct list_head itd_list; /* list of stream's itds */
/* for now, only one hw_transaction per itd */ /* any/all hw_transactions here may be used by that urb */
u32 transaction; unsigned frame; /* where scheduled */
u16 index; /* in urb->iso_frame_desc */ unsigned pg;
u16 uframe; /* in periodic schedule */ unsigned index[8]; /* in urb->iso_frame_desc */
u16 usecs; u8 usecs[8];
} __attribute__ ((aligned (32))); } __attribute__ ((aligned (32)));
/*-------------------------------------------------------------------------*/ /*-------------------------------------------------------------------------*/
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7