diff options
Diffstat (limited to 'drivers/usb/host/xhci-ring.c')
-rw-r--r-- | drivers/usb/host/xhci-ring.c | 1437 |
1 files changed, 1079 insertions, 358 deletions
diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c index bfc99a939455..9f3115e729b1 100644 --- a/drivers/usb/host/xhci-ring.c +++ b/drivers/usb/host/xhci-ring.c @@ -68,6 +68,10 @@ #include <linux/slab.h> #include "xhci.h" +static int handle_cmd_in_cmd_wait_list(struct xhci_hcd *xhci, + struct xhci_virt_device *virt_dev, + struct xhci_event_cmd *event); + /* * Returns zero if the TRB isn't in this segment, otherwise it returns the DMA * address of the TRB. @@ -131,7 +135,7 @@ static void next_trb(struct xhci_hcd *xhci, *seg = (*seg)->next; *trb = ((*seg)->trbs); } else { - *trb = (*trb)++; + (*trb)++; } } @@ -301,28 +305,6 @@ static int room_on_ring(struct xhci_hcd *xhci, struct xhci_ring *ring, return 1; } -void xhci_set_hc_event_deq(struct xhci_hcd *xhci) -{ - u64 temp; - dma_addr_t deq; - - deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg, - xhci->event_ring->dequeue); - if (deq == 0 && !in_interrupt()) - xhci_warn(xhci, "WARN something wrong with SW event ring " - "dequeue ptr.\n"); - /* Update HC event ring dequeue pointer */ - temp = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue); - temp &= ERST_PTR_MASK; - /* Don't clear the EHB bit (which is RW1C) because - * there might be more events to service. - */ - temp &= ~ERST_EHB; - xhci_dbg(xhci, "// Write event ring dequeue pointer, preserving EHB bit\n"); - xhci_write_64(xhci, ((u64) deq & (u64) ~ERST_PTR_MASK) | temp, - &xhci->ir_set->erst_dequeue); -} - /* Ring the host controller doorbell after placing a command on the ring */ void xhci_ring_cmd_db(struct xhci_hcd *xhci) { @@ -335,7 +317,7 @@ void xhci_ring_cmd_db(struct xhci_hcd *xhci) xhci_readl(xhci, &xhci->dba->doorbell[0]); } -static void ring_ep_doorbell(struct xhci_hcd *xhci, +void xhci_ring_ep_doorbell(struct xhci_hcd *xhci, unsigned int slot_id, unsigned int ep_index, unsigned int stream_id) @@ -359,11 +341,6 @@ static void ring_ep_doorbell(struct xhci_hcd *xhci, field = xhci_readl(xhci, db_addr) & DB_MASK; field |= EPI_TO_DB(ep_index) | STREAM_ID_TO_DB(stream_id); xhci_writel(xhci, field, db_addr); - /* Flush PCI posted writes - FIXME Matthew Wilcox says this - * isn't time-critical and we shouldn't make the CPU wait for - * the flush. - */ - xhci_readl(xhci, db_addr); } } @@ -380,7 +357,7 @@ static void ring_doorbell_for_active_rings(struct xhci_hcd *xhci, /* A ring has pending URBs if its TD list is not empty */ if (!(ep->ep_state & EP_HAS_STREAMS)) { if (!(list_empty(&ep->ring->td_list))) - ring_ep_doorbell(xhci, slot_id, ep_index, 0); + xhci_ring_ep_doorbell(xhci, slot_id, ep_index, 0); return; } @@ -388,7 +365,8 @@ static void ring_doorbell_for_active_rings(struct xhci_hcd *xhci, stream_id++) { struct xhci_stream_info *stream_info = ep->stream_info; if (!list_empty(&stream_info->stream_rings[stream_id]->td_list)) - ring_ep_doorbell(xhci, slot_id, ep_index, stream_id); + xhci_ring_ep_doorbell(xhci, slot_id, ep_index, + stream_id); } } @@ -419,6 +397,50 @@ static struct xhci_segment *find_trb_seg( return cur_seg; } + +static struct xhci_ring *xhci_triad_to_transfer_ring(struct xhci_hcd *xhci, + unsigned int slot_id, unsigned int ep_index, + unsigned int stream_id) +{ + struct xhci_virt_ep *ep; + + ep = &xhci->devs[slot_id]->eps[ep_index]; + /* Common case: no streams */ + if (!(ep->ep_state & EP_HAS_STREAMS)) + return ep->ring; + + if (stream_id == 0) { + xhci_warn(xhci, + "WARN: Slot ID %u, ep index %u has streams, " + "but URB has no stream ID.\n", + slot_id, ep_index); + return NULL; + } + + if (stream_id < ep->stream_info->num_streams) + return ep->stream_info->stream_rings[stream_id]; + + xhci_warn(xhci, + "WARN: Slot ID %u, ep index %u has " + "stream IDs 1 to %u allocated, " + "but stream ID %u is requested.\n", + slot_id, ep_index, + ep->stream_info->num_streams - 1, + stream_id); + return NULL; +} + +/* Get the right ring for the given URB. + * If the endpoint supports streams, boundary check the URB's stream ID. + * If the endpoint doesn't support streams, return the singular endpoint ring. + */ +static struct xhci_ring *xhci_urb_to_transfer_ring(struct xhci_hcd *xhci, + struct urb *urb) +{ + return xhci_triad_to_transfer_ring(xhci, urb->dev->slot_id, + xhci_get_endpoint_index(&urb->ep->desc), urb->stream_id); +} + /* * Move the xHC's endpoint ring dequeue pointer past cur_td. * Record the new state of the xHC's endpoint ring dequeue segment, @@ -578,16 +600,24 @@ static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci, struct xhci_td *cur_td, int status, char *adjective) { struct usb_hcd *hcd = xhci_to_hcd(xhci); + struct urb *urb; + struct urb_priv *urb_priv; - cur_td->urb->hcpriv = NULL; - usb_hcd_unlink_urb_from_ep(hcd, cur_td->urb); - xhci_dbg(xhci, "Giveback %s URB %p\n", adjective, cur_td->urb); + urb = cur_td->urb; + urb_priv = urb->hcpriv; + urb_priv->td_cnt++; - spin_unlock(&xhci->lock); - usb_hcd_giveback_urb(hcd, cur_td->urb, status); - kfree(cur_td); - spin_lock(&xhci->lock); - xhci_dbg(xhci, "%s URB given back\n", adjective); + /* Only giveback urb when this is the last td in urb */ + if (urb_priv->td_cnt == urb_priv->length) { + usb_hcd_unlink_urb_from_ep(hcd, urb); + xhci_dbg(xhci, "Giveback %s URB %p\n", adjective, urb); + + spin_unlock(&xhci->lock); + usb_hcd_giveback_urb(hcd, urb, status); + xhci_urb_free_priv(xhci, urb_priv); + spin_lock(&xhci->lock); + xhci_dbg(xhci, "%s URB given back\n", adjective); + } } /* @@ -601,10 +631,11 @@ static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci, * bit cleared) so that the HW will skip over them. */ static void handle_stopped_endpoint(struct xhci_hcd *xhci, - union xhci_trb *trb) + union xhci_trb *trb, struct xhci_event_cmd *event) { unsigned int slot_id; unsigned int ep_index; + struct xhci_virt_device *virt_dev; struct xhci_ring *ep_ring; struct xhci_virt_ep *ep; struct list_head *entry; @@ -613,6 +644,21 @@ static void handle_stopped_endpoint(struct xhci_hcd *xhci, struct xhci_dequeue_state deq_state; + if (unlikely(TRB_TO_SUSPEND_PORT( + xhci->cmd_ring->dequeue->generic.field[3]))) { + slot_id = TRB_TO_SLOT_ID( + xhci->cmd_ring->dequeue->generic.field[3]); + virt_dev = xhci->devs[slot_id]; + if (virt_dev) + handle_cmd_in_cmd_wait_list(xhci, virt_dev, + event); + else + xhci_warn(xhci, "Stop endpoint command " + "completion for disabled slot %u\n", + slot_id); + return; + } + memset(&deq_state, 0, sizeof(deq_state)); slot_id = TRB_TO_SLOT_ID(trb->generic.field[3]); ep_index = TRB_TO_EP_INDEX(trb->generic.field[3]); @@ -1066,7 +1112,7 @@ bandwidth_change: complete(&xhci->addr_dev); break; case TRB_TYPE(TRB_STOP_RING): - handle_stopped_endpoint(xhci, xhci->cmd_ring->dequeue); + handle_stopped_endpoint(xhci, xhci->cmd_ring->dequeue, event); break; case TRB_TYPE(TRB_SET_DEQ): handle_set_deq_completion(xhci, event, xhci->cmd_ring->dequeue); @@ -1119,20 +1165,74 @@ static void handle_vendor_event(struct xhci_hcd *xhci, static void handle_port_status(struct xhci_hcd *xhci, union xhci_trb *event) { + struct usb_hcd *hcd = xhci_to_hcd(xhci); u32 port_id; + u32 temp, temp1; + u32 __iomem *addr; + int ports; + int slot_id; /* Port status change events always have a successful completion code */ if (GET_COMP_CODE(event->generic.field[2]) != COMP_SUCCESS) { xhci_warn(xhci, "WARN: xHC returned failed port status event\n"); xhci->error_bitmask |= 1 << 8; } - /* FIXME: core doesn't care about all port link state changes yet */ port_id = GET_PORT_ID(event->generic.field[0]); xhci_dbg(xhci, "Port Status Change Event for port %d\n", port_id); + ports = HCS_MAX_PORTS(xhci->hcs_params1); + if ((port_id <= 0) || (port_id > ports)) { + xhci_warn(xhci, "Invalid port id %d\n", port_id); + goto cleanup; + } + + addr = &xhci->op_regs->port_status_base + NUM_PORT_REGS * (port_id - 1); + temp = xhci_readl(xhci, addr); + if ((temp & PORT_CONNECT) && (hcd->state == HC_STATE_SUSPENDED)) { + xhci_dbg(xhci, "resume root hub\n"); + usb_hcd_resume_root_hub(hcd); + } + + if ((temp & PORT_PLC) && (temp & PORT_PLS_MASK) == XDEV_RESUME) { + xhci_dbg(xhci, "port resume event for port %d\n", port_id); + + temp1 = xhci_readl(xhci, &xhci->op_regs->command); + if (!(temp1 & CMD_RUN)) { + xhci_warn(xhci, "xHC is not running.\n"); + goto cleanup; + } + + if (DEV_SUPERSPEED(temp)) { + xhci_dbg(xhci, "resume SS port %d\n", port_id); + temp = xhci_port_state_to_neutral(temp); + temp &= ~PORT_PLS_MASK; + temp |= PORT_LINK_STROBE | XDEV_U0; + xhci_writel(xhci, temp, addr); + slot_id = xhci_find_slot_id_by_port(xhci, port_id); + if (!slot_id) { + xhci_dbg(xhci, "slot_id is zero\n"); + goto cleanup; + } + xhci_ring_device(xhci, slot_id); + xhci_dbg(xhci, "resume SS port %d finished\n", port_id); + /* Clear PORT_PLC */ + temp = xhci_readl(xhci, addr); + temp = xhci_port_state_to_neutral(temp); + temp |= PORT_PLC; + xhci_writel(xhci, temp, addr); + } else { + xhci_dbg(xhci, "resume HS port %d\n", port_id); + xhci->resume_done[port_id - 1] = jiffies + + msecs_to_jiffies(20); + mod_timer(&hcd->rh_timer, + xhci->resume_done[port_id - 1]); + /* Do the rest in GetPortStatus */ + } + } + +cleanup: /* Update event ring dequeue pointer before dropping the lock */ inc_deq(xhci, xhci->event_ring, true); - xhci_set_hc_event_deq(xhci); spin_unlock(&xhci->lock); /* Pass this up to the core */ @@ -1258,6 +1358,425 @@ int xhci_is_vendor_info_code(struct xhci_hcd *xhci, unsigned int trb_comp_code) } /* + * Finish the td processing, remove the td from td list; + * Return 1 if the urb can be given back. + */ +static int finish_td(struct xhci_hcd *xhci, struct xhci_td *td, + union xhci_trb *event_trb, struct xhci_transfer_event *event, + struct xhci_virt_ep *ep, int *status, bool skip) +{ + struct xhci_virt_device *xdev; + struct xhci_ring *ep_ring; + unsigned int slot_id; + int ep_index; + struct urb *urb = NULL; + struct xhci_ep_ctx *ep_ctx; + int ret = 0; + struct urb_priv *urb_priv; + u32 trb_comp_code; + + slot_id = TRB_TO_SLOT_ID(event->flags); + xdev = xhci->devs[slot_id]; + ep_index = TRB_TO_EP_ID(event->flags) - 1; + ep_ring = xhci_dma_to_transfer_ring(ep, event->buffer); + ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); + trb_comp_code = GET_COMP_CODE(event->transfer_len); + + if (skip) + goto td_cleanup; + + if (trb_comp_code == COMP_STOP_INVAL || + trb_comp_code == COMP_STOP) { + /* The Endpoint Stop Command completion will take care of any + * stopped TDs. A stopped TD may be restarted, so don't update + * the ring dequeue pointer or take this TD off any lists yet. + */ + ep->stopped_td = td; + ep->stopped_trb = event_trb; + return 0; + } else { + if (trb_comp_code == COMP_STALL) { + /* The transfer is completed from the driver's + * perspective, but we need to issue a set dequeue + * command for this stalled endpoint to move the dequeue + * pointer past the TD. We can't do that here because + * the halt condition must be cleared first. Let the + * USB class driver clear the stall later. + */ + ep->stopped_td = td; + ep->stopped_trb = event_trb; + ep->stopped_stream = ep_ring->stream_id; + } else if (xhci_requires_manual_halt_cleanup(xhci, + ep_ctx, trb_comp_code)) { + /* Other types of errors halt the endpoint, but the + * class driver doesn't call usb_reset_endpoint() unless + * the error is -EPIPE. Clear the halted status in the + * xHCI hardware manually. + */ + xhci_cleanup_halted_endpoint(xhci, + slot_id, ep_index, ep_ring->stream_id, + td, event_trb); + } else { + /* Update ring dequeue pointer */ + while (ep_ring->dequeue != td->last_trb) + inc_deq(xhci, ep_ring, false); + inc_deq(xhci, ep_ring, false); + } + +td_cleanup: + /* Clean up the endpoint's TD list */ + urb = td->urb; + urb_priv = urb->hcpriv; + + /* Do one last check of the actual transfer length. + * If the host controller said we transferred more data than + * the buffer length, urb->actual_length will be a very big + * number (since it's unsigned). Play it safe and say we didn't + * transfer anything. + */ + if (urb->actual_length > urb->transfer_buffer_length) { + xhci_warn(xhci, "URB transfer length is wrong, " + "xHC issue? req. len = %u, " + "act. len = %u\n", + urb->transfer_buffer_length, + urb->actual_length); + urb->actual_length = 0; + if (td->urb->transfer_flags & URB_SHORT_NOT_OK) + *status = -EREMOTEIO; + else + *status = 0; + } + list_del(&td->td_list); + /* Was this TD slated to be cancelled but completed anyway? */ + if (!list_empty(&td->cancelled_td_list)) + list_del(&td->cancelled_td_list); + + urb_priv->td_cnt++; + /* Giveback the urb when all the tds are completed */ + if (urb_priv->td_cnt == urb_priv->length) + ret = 1; + } + + return ret; +} + +/* + * Process control tds, update urb status and actual_length. + */ +static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_td *td, + union xhci_trb *event_trb, struct xhci_transfer_event *event, + struct xhci_virt_ep *ep, int *status) +{ + struct xhci_virt_device *xdev; + struct xhci_ring *ep_ring; + unsigned int slot_id; + int ep_index; + struct xhci_ep_ctx *ep_ctx; + u32 trb_comp_code; + + slot_id = TRB_TO_SLOT_ID(event->flags); + xdev = xhci->devs[slot_id]; + ep_index = TRB_TO_EP_ID(event->flags) - 1; + ep_ring = xhci_dma_to_transfer_ring(ep, event->buffer); + ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); + trb_comp_code = GET_COMP_CODE(event->transfer_len); + + xhci_debug_trb(xhci, xhci->event_ring->dequeue); + switch (trb_comp_code) { + case COMP_SUCCESS: + if (event_trb == ep_ring->dequeue) { + xhci_warn(xhci, "WARN: Success on ctrl setup TRB " + "without IOC set??\n"); + *status = -ESHUTDOWN; + } else if (event_trb != td->last_trb) { + xhci_warn(xhci, "WARN: Success on ctrl data TRB " + "without IOC set??\n"); + *status = -ESHUTDOWN; + } else { + xhci_dbg(xhci, "Successful control transfer!\n"); + *status = 0; + } + break; + case COMP_SHORT_TX: + xhci_warn(xhci, "WARN: short transfer on control ep\n"); + if (td->urb->transfer_flags & URB_SHORT_NOT_OK) + *status = -EREMOTEIO; + else + *status = 0; + break; + default: + if (!xhci_requires_manual_halt_cleanup(xhci, + ep_ctx, trb_comp_code)) + break; + xhci_dbg(xhci, "TRB error code %u, " + "halted endpoint index = %u\n", + trb_comp_code, ep_index); + /* else fall through */ + case COMP_STALL: + /* Did we transfer part of the data (middle) phase? */ + if (event_trb != ep_ring->dequeue && + event_trb != td->last_trb) + td->urb->actual_length = + td->urb->transfer_buffer_length + - TRB_LEN(event->transfer_len); + else + td->urb->actual_length = 0; + + xhci_cleanup_halted_endpoint(xhci, + slot_id, ep_index, 0, td, event_trb); + return finish_td(xhci, td, event_trb, event, ep, status, true); + } + /* + * Did we transfer any data, despite the errors that might have + * happened? I.e. did we get past the setup stage? + */ + if (event_trb != ep_ring->dequeue) { + /* The event was for the status stage */ + if (event_trb == td->last_trb) { + if (td->urb->actual_length != 0) { + /* Don't overwrite a previously set error code + */ + if ((*status == -EINPROGRESS || *status == 0) && + (td->urb->transfer_flags + & URB_SHORT_NOT_OK)) + /* Did we already see a short data + * stage? */ + *status = -EREMOTEIO; + } else { + td->urb->actual_length = + td->urb->transfer_buffer_length; + } + } else { + /* Maybe the event was for the data stage? */ + if (trb_comp_code != COMP_STOP_INVAL) { + /* We didn't stop on a link TRB in the middle */ + td->urb->actual_length = + td->urb->transfer_buffer_length - + TRB_LEN(event->transfer_len); + xhci_dbg(xhci, "Waiting for status " + "stage event\n"); + return 0; + } + } + } + + return finish_td(xhci, td, event_trb, event, ep, status, false); +} + +/* + * Process isochronous tds, update urb packet status and actual_length. + */ +static int process_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td, + union xhci_trb *event_trb, struct xhci_transfer_event *event, + struct xhci_virt_ep *ep, int *status) +{ + struct xhci_ring *ep_ring; + struct urb_priv *urb_priv; + int idx; + int len = 0; + int skip_td = 0; + union xhci_trb *cur_trb; + struct xhci_segment *cur_seg; + u32 trb_comp_code; + + ep_ring = xhci_dma_to_transfer_ring(ep, event->buffer); + trb_comp_code = GET_COMP_CODE(event->transfer_len); + urb_priv = td->urb->hcpriv; + idx = urb_priv->td_cnt; + + if (ep->skip) { + /* The transfer is partly done */ + *status = -EXDEV; + td->urb->iso_frame_desc[idx].status = -EXDEV; + } else { + /* handle completion code */ + switch (trb_comp_code) { + case COMP_SUCCESS: + td->urb->iso_frame_desc[idx].status = 0; + xhci_dbg(xhci, "Successful isoc transfer!\n"); + break; + case COMP_SHORT_TX: + if (td->urb->transfer_flags & URB_SHORT_NOT_OK) + td->urb->iso_frame_desc[idx].status = + -EREMOTEIO; + else + td->urb->iso_frame_desc[idx].status = 0; + break; + case COMP_BW_OVER: + td->urb->iso_frame_desc[idx].status = -ECOMM; + skip_td = 1; + break; + case COMP_BUFF_OVER: + case COMP_BABBLE: + td->urb->iso_frame_desc[idx].status = -EOVERFLOW; + skip_td = 1; + break; + case COMP_STALL: + td->urb->iso_frame_desc[idx].status = -EPROTO; + skip_td = 1; + break; + case COMP_STOP: + case COMP_STOP_INVAL: + break; + default: + td->urb->iso_frame_desc[idx].status = -1; + break; + } + } + + /* calc actual length */ + if (ep->skip) { + td->urb->iso_frame_desc[idx].actual_length = 0; + /* Update ring dequeue pointer */ + while (ep_ring->dequeue != td->last_trb) + inc_deq(xhci, ep_ring, false); + inc_deq(xhci, ep_ring, false); + return finish_td(xhci, td, event_trb, event, ep, status, true); + } + + if (trb_comp_code == COMP_SUCCESS || skip_td == 1) { + td->urb->iso_frame_desc[idx].actual_length = + td->urb->iso_frame_desc[idx].length; + td->urb->actual_length += + td->urb->iso_frame_desc[idx].length; + } else { + for (cur_trb = ep_ring->dequeue, + cur_seg = ep_ring->deq_seg; cur_trb != event_trb; + next_trb(xhci, ep_ring, &cur_seg, &cur_trb)) { + if ((cur_trb->generic.field[3] & + TRB_TYPE_BITMASK) != TRB_TYPE(TRB_TR_NOOP) && + (cur_trb->generic.field[3] & + TRB_TYPE_BITMASK) != TRB_TYPE(TRB_LINK)) + len += + TRB_LEN(cur_trb->generic.field[2]); + } + len += TRB_LEN(cur_trb->generic.field[2]) - + TRB_LEN(event->transfer_len); + + if (trb_comp_code != COMP_STOP_INVAL) { + td->urb->iso_frame_desc[idx].actual_length = len; + td->urb->actual_length += len; + } + } + + if ((idx == urb_priv->length - 1) && *status == -EINPROGRESS) + *status = 0; + + return finish_td(xhci, td, event_trb, event, ep, status, false); +} + +/* + * Process bulk and interrupt tds, update urb status and actual_length. + */ +static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_td *td, + union xhci_trb *event_trb, struct xhci_transfer_event *event, + struct xhci_virt_ep *ep, int *status) +{ + struct xhci_ring *ep_ring; + union xhci_trb *cur_trb; + struct xhci_segment *cur_seg; + u32 trb_comp_code; + + ep_ring = xhci_dma_to_transfer_ring(ep, event->buffer); + trb_comp_code = GET_COMP_CODE(event->transfer_len); + + switch (trb_comp_code) { + case COMP_SUCCESS: + /* Double check that the HW transferred everything. */ + if (event_trb != td->last_trb) { + xhci_warn(xhci, "WARN Successful completion " + "on short TX\n"); + if (td->urb->transfer_flags & URB_SHORT_NOT_OK) + *status = -EREMOTEIO; + else + *status = 0; + } else { + if (usb_endpoint_xfer_bulk(&td->urb->ep->desc)) + xhci_dbg(xhci, "Successful bulk " + "transfer!\n"); + else + xhci_dbg(xhci, "Successful interrupt " + "transfer!\n"); + *status = 0; + } + break; + case COMP_SHORT_TX: + if (td->urb->transfer_flags & URB_SHORT_NOT_OK) + *status = -EREMOTEIO; + else + *status = 0; + break; + default: + /* Others already handled above */ + break; + } + dev_dbg(&td->urb->dev->dev, + "ep %#x - asked for %d bytes, " + "%d bytes untransferred\n", + td->urb->ep->desc.bEndpointAddress, + td->urb->transfer_buffer_length, + TRB_LEN(event->transfer_len)); + /* Fast path - was this the last TRB in the TD for this URB? */ + if (event_trb == td->last_trb) { + if (TRB_LEN(event->transfer_len) != 0) { + td->urb->actual_length = + td->urb->transfer_buffer_length - + TRB_LEN(event->transfer_len); + if (td->urb->transfer_buffer_length < + td->urb->actual_length) { + xhci_warn(xhci, "HC gave bad length " + "of %d bytes left\n", + TRB_LEN(event->transfer_len)); + td->urb->actual_length = 0; + if (td->urb->transfer_flags & URB_SHORT_NOT_OK) + *status = -EREMOTEIO; + else + *status = 0; + } + /* Don't overwrite a previously set error code */ + if (*status == -EINPROGRESS) { + if (td->urb->transfer_flags & URB_SHORT_NOT_OK) + *status = -EREMOTEIO; + else + *status = 0; + } + } else { + td->urb->actual_length = + td->urb->transfer_buffer_length; + /* Ignore a short packet completion if the + * untransferred length was zero. + */ + if (*status == -EREMOTEIO) + *status = 0; + } + } else { + /* Slow path - walk the list, starting from the dequeue + * pointer, to get the actual length transferred. + */ + td->urb->actual_length = 0; + for (cur_trb = ep_ring->dequeue, cur_seg = ep_ring->deq_seg; + cur_trb != event_trb; + next_trb(xhci, ep_ring, &cur_seg, &cur_trb)) { + if ((cur_trb->generic.field[3] & + TRB_TYPE_BITMASK) != TRB_TYPE(TRB_TR_NOOP) && + (cur_trb->generic.field[3] & + TRB_TYPE_BITMASK) != TRB_TYPE(TRB_LINK)) + td->urb->actual_length += + TRB_LEN(cur_trb->generic.field[2]); + } + /* If the ring didn't stop on a Link or No-op TRB, add + * in the actual bytes transferred from the Normal TRB + */ + if (trb_comp_code != COMP_STOP_INVAL) + td->urb->actual_length += + TRB_LEN(cur_trb->generic.field[2]) - + TRB_LEN(event->transfer_len); + } + + return finish_td(xhci, td, event_trb, event, ep, status, false); +} + +/* * If this function returns an error condition, it means it got a Transfer * event with a corrupted Slot ID, Endpoint ID, or TRB DMA address. * At this point, the host controller is probably hosed and should be reset. @@ -1276,10 +1795,11 @@ static int handle_tx_event(struct xhci_hcd *xhci, union xhci_trb *event_trb; struct urb *urb = NULL; int status = -EINPROGRESS; + struct urb_priv *urb_priv; struct xhci_ep_ctx *ep_ctx; u32 trb_comp_code; + int ret = 0; - xhci_dbg(xhci, "In %s\n", __func__); slot_id = TRB_TO_SLOT_ID(event->flags); xdev = xhci->devs[slot_id]; if (!xdev) { @@ -1293,51 +1813,16 @@ static int handle_tx_event(struct xhci_hcd *xhci, ep = &xdev->eps[ep_index]; ep_ring = xhci_dma_to_transfer_ring(ep, event->buffer); ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); - if (!ep_ring || (ep_ctx->ep_info & EP_STATE_MASK) == EP_STATE_DISABLED) { + if (!ep_ring || + (ep_ctx->ep_info & EP_STATE_MASK) == EP_STATE_DISABLED) { xhci_err(xhci, "ERROR Transfer event for disabled endpoint " "or incorrect stream ring\n"); return -ENODEV; } event_dma = event->buffer; - /* This TRB should be in the TD at the head of this ring's TD list */ - xhci_dbg(xhci, "%s - checking for list empty\n", __func__); - if (list_empty(&ep_ring->td_list)) { - xhci_warn(xhci, "WARN Event TRB for slot %d ep %d with no TDs queued?\n", - TRB_TO_SLOT_ID(event->flags), ep_index); - xhci_dbg(xhci, "Event TRB with TRB type ID %u\n", - (unsigned int) (event->flags & TRB_TYPE_BITMASK)>>10); - xhci_print_trb_offsets(xhci, (union xhci_trb *) event); - urb = NULL; - goto cleanup; - } - xhci_dbg(xhci, "%s - getting list entry\n", __func__); - td = list_entry(ep_ring->td_list.next, struct xhci_td, td_list); - - /* Is this a TRB in the currently executing TD? */ - xhci_dbg(xhci, "%s - looking for TD\n", __func__); - event_seg = trb_in_td(ep_ring->deq_seg, ep_ring->dequeue, - td->last_trb, event_dma); - xhci_dbg(xhci, "%s - found event_seg = %p\n", __func__, event_seg); - if (!event_seg) { - /* HC is busted, give up! */ - xhci_err(xhci, "ERROR Transfer event TRB DMA ptr not part of current TD\n"); - return -ESHUTDOWN; - } - event_trb = &event_seg->trbs[(event_dma - event_seg->dma) / sizeof(*event_trb)]; - xhci_dbg(xhci, "Event TRB with TRB type ID %u\n", - (unsigned int) (event->flags & TRB_TYPE_BITMASK)>>10); - xhci_dbg(xhci, "Offset 0x00 (buffer lo) = 0x%x\n", - lower_32_bits(event->buffer)); - xhci_dbg(xhci, "Offset 0x04 (buffer hi) = 0x%x\n", - upper_32_bits(event->buffer)); - xhci_dbg(xhci, "Offset 0x08 (transfer length) = 0x%x\n", - (unsigned int) event->transfer_len); - xhci_dbg(xhci, "Offset 0x0C (flags) = 0x%x\n", - (unsigned int) event->flags); - - /* Look for common error cases */ trb_comp_code = GET_COMP_CODE(event->transfer_len); + /* Look for common error cases */ switch (trb_comp_code) { /* Skip codes that require special handling depending on * transfer type @@ -1373,278 +1858,156 @@ static int handle_tx_event(struct xhci_hcd *xhci, xhci_warn(xhci, "WARN: HC couldn't access mem fast enough\n"); status = -ENOSR; break; + case COMP_BW_OVER: + xhci_warn(xhci, "WARN: bandwidth overrun event on endpoint\n"); + break; + case COMP_BUFF_OVER: + xhci_warn(xhci, "WARN: buffer overrun event on endpoint\n"); + break; + case COMP_UNDERRUN: + /* + * When the Isoch ring is empty, the xHC will generate + * a Ring Overrun Event for IN Isoch endpoint or Ring + * Underrun Event for OUT Isoch endpoint. + */ + xhci_dbg(xhci, "underrun event on endpoint\n"); + if (!list_empty(&ep_ring->td_list)) + xhci_dbg(xhci, "Underrun Event for slot %d ep %d " + "still with TDs queued?\n", + TRB_TO_SLOT_ID(event->flags), ep_index); + goto cleanup; + case COMP_OVERRUN: + xhci_dbg(xhci, "overrun event on endpoint\n"); + if (!list_empty(&ep_ring->td_list)) + xhci_dbg(xhci, "Overrun Event for slot %d ep %d " + "still with TDs queued?\n", + TRB_TO_SLOT_ID(event->flags), ep_index); + goto cleanup; + case COMP_MISSED_INT: + /* + * When encounter missed service error, one or more isoc tds + * may be missed by xHC. + * Set skip flag of the ep_ring; Complete the missed tds as + * short transfer when process the ep_ring next time. + */ + ep->skip = true; + xhci_dbg(xhci, "Miss service interval error, set skip flag\n"); + goto cleanup; default: if (xhci_is_vendor_info_code(xhci, trb_comp_code)) { status = 0; break; } - xhci_warn(xhci, "ERROR Unknown event condition, HC probably busted\n"); - urb = NULL; + xhci_warn(xhci, "ERROR Unknown event condition, HC probably " + "busted\n"); goto cleanup; } - /* Now update the urb's actual_length and give back to the core */ - /* Was this a control transfer? */ - if (usb_endpoint_xfer_control(&td->urb->ep->desc)) { - xhci_debug_trb(xhci, xhci->event_ring->dequeue); - switch (trb_comp_code) { - case COMP_SUCCESS: - if (event_trb == ep_ring->dequeue) { - xhci_warn(xhci, "WARN: Success on ctrl setup TRB without IOC set??\n"); - status = -ESHUTDOWN; - } else if (event_trb != td->last_trb) { - xhci_warn(xhci, "WARN: Success on ctrl data TRB without IOC set??\n"); - status = -ESHUTDOWN; - } else { - xhci_dbg(xhci, "Successful control transfer!\n"); - status = 0; - } - break; - case COMP_SHORT_TX: - xhci_warn(xhci, "WARN: short transfer on control ep\n"); - if (td->urb->transfer_flags & URB_SHORT_NOT_OK) - status = -EREMOTEIO; - else - status = 0; - break; - default: - if (!xhci_requires_manual_halt_cleanup(xhci, - ep_ctx, trb_comp_code)) - break; - xhci_dbg(xhci, "TRB error code %u, " - "halted endpoint index = %u\n", - trb_comp_code, ep_index); - /* else fall through */ - case COMP_STALL: - /* Did we transfer part of the data (middle) phase? */ - if (event_trb != ep_ring->dequeue && - event_trb != td->last_trb) - td->urb->actual_length = - td->urb->transfer_buffer_length - - TRB_LEN(event->transfer_len); - else - td->urb->actual_length = 0; - - xhci_cleanup_halted_endpoint(xhci, - slot_id, ep_index, 0, td, event_trb); - goto td_cleanup; - } - /* - * Did we transfer any data, despite the errors that might have - * happened? I.e. did we get past the setup stage? + do { + /* This TRB should be in the TD at the head of this ring's + * TD list. */ - if (event_trb != ep_ring->dequeue) { - /* The event was for the status stage */ - if (event_trb == td->last_trb) { - if (td->urb->actual_length != 0) { - /* Don't overwrite a previously set error code */ - if ((status == -EINPROGRESS || - status == 0) && - (td->urb->transfer_flags - & URB_SHORT_NOT_OK)) - /* Did we already see a short data stage? */ - status = -EREMOTEIO; - } else { - td->urb->actual_length = - td->urb->transfer_buffer_length; - } - } else { - /* Maybe the event was for the data stage? */ - if (trb_comp_code != COMP_STOP_INVAL) { - /* We didn't stop on a link TRB in the middle */ - td->urb->actual_length = - td->urb->transfer_buffer_length - - TRB_LEN(event->transfer_len); - xhci_dbg(xhci, "Waiting for status stage event\n"); - urb = NULL; - goto cleanup; - } + if (list_empty(&ep_ring->td_list)) { + xhci_warn(xhci, "WARN Event TRB for slot %d ep %d " + "with no TDs queued?\n", + TRB_TO_SLOT_ID(event->flags), ep_index); + xhci_dbg(xhci, "Event TRB with TRB type ID %u\n", + (unsigned int) (event->flags & TRB_TYPE_BITMASK)>>10); + xhci_print_trb_offsets(xhci, (union xhci_trb *) event); + if (ep->skip) { + ep->skip = false; + xhci_dbg(xhci, "td_list is empty while skip " + "flag set. Clear skip flag.\n"); } + ret = 0; + goto cleanup; } - } else { - switch (trb_comp_code) { - case COMP_SUCCESS: - /* Double check that the HW transferred everything. */ - if (event_trb != td->last_trb) { - xhci_warn(xhci, "WARN Successful completion " - "on short TX\n"); - if (td->urb->transfer_flags & URB_SHORT_NOT_OK) - status = -EREMOTEIO; - else - status = 0; - } else { - if (usb_endpoint_xfer_bulk(&td->urb->ep->desc)) - xhci_dbg(xhci, "Successful bulk " - "transfer!\n"); - else - xhci_dbg(xhci, "Successful interrupt " - "transfer!\n"); - status = 0; - } - break; - case COMP_SHORT_TX: - if (td->urb->transfer_flags & URB_SHORT_NOT_OK) - status = -EREMOTEIO; - else - status = 0; - break; - default: - /* Others already handled above */ - break; + + td = list_entry(ep_ring->td_list.next, struct xhci_td, td_list); + /* Is this a TRB in the currently executing TD? */ + event_seg = trb_in_td(ep_ring->deq_seg, ep_ring->dequeue, + td->last_trb, event_dma); + if (event_seg && ep->skip) { + xhci_dbg(xhci, "Found td. Clear skip flag.\n"); + ep->skip = false; + } + if (!event_seg && + (!ep->skip || !usb_endpoint_xfer_isoc(&td->urb->ep->desc))) { + /* HC is busted, give up! */ + xhci_err(xhci, "ERROR Transfer event TRB DMA ptr not " + "part of current TD\n"); + return -ESHUTDOWN; } - dev_dbg(&td->urb->dev->dev, - "ep %#x - asked for %d bytes, " - "%d bytes untransferred\n", - td->urb->ep->desc.bEndpointAddress, - td->urb->transfer_buffer_length, - TRB_LEN(event->transfer_len)); - /* Fast path - was this the last TRB in the TD for this URB? */ - if (event_trb == td->last_trb) { - if (TRB_LEN(event->transfer_len) != 0) { - td->urb->actual_length = - td->urb->transfer_buffer_length - - TRB_LEN(event->transfer_len); - if (td->urb->transfer_buffer_length < - td->urb->actual_length) { - xhci_warn(xhci, "HC gave bad length " - "of %d bytes left\n", - TRB_LEN(event->transfer_len)); - td->urb->actual_length = 0; - if (td->urb->transfer_flags & - URB_SHORT_NOT_OK) - status = -EREMOTEIO; - else - status = 0; - } - /* Don't overwrite a previously set error code */ - if (status == -EINPROGRESS) { - if (td->urb->transfer_flags & URB_SHORT_NOT_OK) - status = -EREMOTEIO; - else - status = 0; - } - } else { - td->urb->actual_length = td->urb->transfer_buffer_length; - /* Ignore a short packet completion if the - * untransferred length was zero. - */ - if (status == -EREMOTEIO) - status = 0; - } - } else { - /* Slow path - walk the list, starting from the dequeue - * pointer, to get the actual length transferred. - */ - union xhci_trb *cur_trb; - struct xhci_segment *cur_seg; - td->urb->actual_length = 0; - for (cur_trb = ep_ring->dequeue, cur_seg = ep_ring->deq_seg; - cur_trb != event_trb; - next_trb(xhci, ep_ring, &cur_seg, &cur_trb)) { - if ((cur_trb->generic.field[3] & - TRB_TYPE_BITMASK) != TRB_TYPE(TRB_TR_NOOP) && - (cur_trb->generic.field[3] & - TRB_TYPE_BITMASK) != TRB_TYPE(TRB_LINK)) - td->urb->actual_length += - TRB_LEN(cur_trb->generic.field[2]); - } - /* If the ring didn't stop on a Link or No-op TRB, add - * in the actual bytes transferred from the Normal TRB + if (event_seg) { + event_trb = &event_seg->trbs[(event_dma - + event_seg->dma) / sizeof(*event_trb)]; + /* + * No-op TRB should not trigger interrupts. + * If event_trb is a no-op TRB, it means the + * corresponding TD has been cancelled. Just ignore + * the TD. */ - if (trb_comp_code != COMP_STOP_INVAL) - td->urb->actual_length += - TRB_LEN(cur_trb->generic.field[2]) - - TRB_LEN(event->transfer_len); + if ((event_trb->generic.field[3] & TRB_TYPE_BITMASK) + == TRB_TYPE(TRB_TR_NOOP)) { + xhci_dbg(xhci, "event_trb is a no-op TRB. " + "Skip it\n"); + goto cleanup; + } } - } - if (trb_comp_code == COMP_STOP_INVAL || - trb_comp_code == COMP_STOP) { - /* The Endpoint Stop Command completion will take care of any - * stopped TDs. A stopped TD may be restarted, so don't update - * the ring dequeue pointer or take this TD off any lists yet. + + /* Now update the urb's actual_length and give back to + * the core */ - ep->stopped_td = td; - ep->stopped_trb = event_trb; - } else { - if (trb_comp_code == COMP_STALL) { - /* The transfer is completed from the driver's - * perspective, but we need to issue a set dequeue - * command for this stalled endpoint to move the dequeue - * pointer past the TD. We can't do that here because - * the halt condition must be cleared first. Let the - * USB class driver clear the stall later. - */ - ep->stopped_td = td; - ep->stopped_trb = event_trb; - ep->stopped_stream = ep_ring->stream_id; - } else if (xhci_requires_manual_halt_cleanup(xhci, - ep_ctx, trb_comp_code)) { - /* Other types of errors halt the endpoint, but the - * class driver doesn't call usb_reset_endpoint() unless - * the error is -EPIPE. Clear the halted status in the - * xHCI hardware manually. - */ - xhci_cleanup_halted_endpoint(xhci, - slot_id, ep_index, ep_ring->stream_id, td, event_trb); - } else { - /* Update ring dequeue pointer */ - while (ep_ring->dequeue != td->last_trb) - inc_deq(xhci, ep_ring, false); - inc_deq(xhci, ep_ring, false); - } + if (usb_endpoint_xfer_control(&td->urb->ep->desc)) + ret = process_ctrl_td(xhci, td, event_trb, event, ep, + &status); + else if (usb_endpoint_xfer_isoc(&td->urb->ep->desc)) + ret = process_isoc_td(xhci, td, event_trb, event, ep, + &status); + else + ret = process_bulk_intr_td(xhci, td, event_trb, event, + ep, &status); -td_cleanup: - /* Clean up the endpoint's TD list */ - urb = td->urb; - /* Do one last check of the actual transfer length. - * If the host controller said we transferred more data than - * the buffer length, urb->actual_length will be a very big - * number (since it's unsigned). Play it safe and say we didn't - * transfer anything. +cleanup: + /* + * Do not update event ring dequeue pointer if ep->skip is set. + * Will roll back to continue process missed tds. */ - if (urb->actual_length > urb->transfer_buffer_length) { - xhci_warn(xhci, "URB transfer length is wrong, " - "xHC issue? req. len = %u, " - "act. len = %u\n", - urb->transfer_buffer_length, - urb->actual_length); - urb->actual_length = 0; - if (td->urb->transfer_flags & URB_SHORT_NOT_OK) - status = -EREMOTEIO; - else - status = 0; + if (trb_comp_code == COMP_MISSED_INT || !ep->skip) { + inc_deq(xhci, xhci->event_ring, true); } - list_del(&td->td_list); - /* Was this TD slated to be cancelled but completed anyway? */ - if (!list_empty(&td->cancelled_td_list)) - list_del(&td->cancelled_td_list); - /* Leave the TD around for the reset endpoint function to use - * (but only if it's not a control endpoint, since we already - * queued the Set TR dequeue pointer command for stalled - * control endpoints). - */ - if (usb_endpoint_xfer_control(&urb->ep->desc) || - (trb_comp_code != COMP_STALL && - trb_comp_code != COMP_BABBLE)) { - kfree(td); + if (ret) { + urb = td->urb; + urb_priv = urb->hcpriv; + /* Leave the TD around for the reset endpoint function + * to use(but only if it's not a control endpoint, + * since we already queued the Set TR dequeue pointer + * command for stalled control endpoints). + */ + if (usb_endpoint_xfer_control(&urb->ep->desc) || + (trb_comp_code != COMP_STALL && + trb_comp_code != COMP_BABBLE)) + xhci_urb_free_priv(xhci, urb_priv); + + usb_hcd_unlink_urb_from_ep(xhci_to_hcd(xhci), urb); + xhci_dbg(xhci, "Giveback URB %p, len = %d, " + "status = %d\n", + urb, urb->actual_length, status); + spin_unlock(&xhci->lock); + usb_hcd_giveback_urb(xhci_to_hcd(xhci), urb, status); + spin_lock(&xhci->lock); } - urb->hcpriv = NULL; - } -cleanup: - inc_deq(xhci, xhci->event_ring, true); - xhci_set_hc_event_deq(xhci); - /* FIXME for multi-TD URBs (who have buffers bigger than 64MB) */ - if (urb) { - usb_hcd_unlink_urb_from_ep(xhci_to_hcd(xhci), urb); - xhci_dbg(xhci, "Giveback URB %p, len = %d, status = %d\n", - urb, urb->actual_length, status); - spin_unlock(&xhci->lock); - usb_hcd_giveback_urb(xhci_to_hcd(xhci), urb, status); - spin_lock(&xhci->lock); - } + /* + * If ep->skip is set, it means there are missed tds on the + * endpoint ring need to take care of. + * Process them as short transfer until reach the td pointed by + * the event. + */ + } while (ep->skip && trb_comp_code != COMP_MISSED_INT); + return 0; } @@ -1652,7 +2015,7 @@ cleanup: * This function handles all OS-owned events on the event ring. It may drop * xhci->lock between event processing (e.g. to pass up port status changes). */ -void xhci_handle_event(struct xhci_hcd *xhci) +static void xhci_handle_event(struct xhci_hcd *xhci) { union xhci_trb *event; int update_ptrs = 1; @@ -1710,15 +2073,130 @@ void xhci_handle_event(struct xhci_hcd *xhci) return; } - if (update_ptrs) { - /* Update SW and HC event ring dequeue pointer */ + if (update_ptrs) + /* Update SW event ring dequeue pointer */ inc_deq(xhci, xhci->event_ring, true); - xhci_set_hc_event_deq(xhci); - } + /* Are there more items on the event ring? */ xhci_handle_event(xhci); } +/* + * xHCI spec says we can get an interrupt, and if the HC has an error condition, + * we might get bad data out of the event ring. Section 4.10.2.7 has a list of + * indicators of an event TRB error, but we check the status *first* to be safe. + */ +irqreturn_t xhci_irq(struct usb_hcd *hcd) +{ + struct xhci_hcd *xhci = hcd_to_xhci(hcd); + u32 status; + union xhci_trb *trb; + u64 temp_64; + union xhci_trb *event_ring_deq; + dma_addr_t deq; + + spin_lock(&xhci->lock); + trb = xhci->event_ring->dequeue; + /* Check if the xHC generated the interrupt, or the irq is shared */ + status = xhci_readl(xhci, &xhci->op_regs->status); + if (status == 0xffffffff) + goto hw_died; + + if (!(status & STS_EINT)) { + spin_unlock(&xhci->lock); + xhci_warn(xhci, "Spurious interrupt.\n"); + return IRQ_NONE; + } + xhci_dbg(xhci, "op reg status = %08x\n", status); + xhci_dbg(xhci, "Event ring dequeue ptr:\n"); + xhci_dbg(xhci, "@%llx %08x %08x %08x %08x\n", + (unsigned long long) + xhci_trb_virt_to_dma(xhci->event_ring->deq_seg, trb), + lower_32_bits(trb->link.segment_ptr), + upper_32_bits(trb->link.segment_ptr), + (unsigned int) trb->link.intr_target, + (unsigned int) trb->link.control); + + if (status & STS_FATAL) { + xhci_warn(xhci, "WARNING: Host System Error\n"); + xhci_halt(xhci); +hw_died: + xhci_to_hcd(xhci)->state = HC_STATE_HALT; + spin_unlock(&xhci->lock); + return -ESHUTDOWN; + } + + /* + * Clear the op reg interrupt status first, + * so we can receive interrupts from other MSI-X interrupters. + * Write 1 to clear the interrupt status. + */ + status |= STS_EINT; + xhci_writel(xhci, status, &xhci->op_regs->status); + /* FIXME when MSI-X is supported and there are multiple vectors */ + /* Clear the MSI-X event interrupt status */ + + if (hcd->irq != -1) { + u32 irq_pending; + /* Acknowledge the PCI interrupt */ + irq_pending = xhci_readl(xhci, &xhci->ir_set->irq_pending); + irq_pending |= 0x3; + xhci_writel(xhci, irq_pending, &xhci->ir_set->irq_pending); + } + + if (xhci->xhc_state & XHCI_STATE_DYING) { + xhci_dbg(xhci, "xHCI dying, ignoring interrupt. " + "Shouldn't IRQs be disabled?\n"); + /* Clear the event handler busy flag (RW1C); + * the event ring should be empty. + */ + temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue); + xhci_write_64(xhci, temp_64 | ERST_EHB, + &xhci->ir_set->erst_dequeue); + spin_unlock(&xhci->lock); + + return IRQ_HANDLED; + } + + event_ring_deq = xhci->event_ring->dequeue; + /* FIXME this should be a delayed service routine + * that clears the EHB. + */ + xhci_handle_event(xhci); + + temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue); + /* If necessary, update the HW's version of the event ring deq ptr. */ + if (event_ring_deq != xhci->event_ring->dequeue) { + deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg, + xhci->event_ring->dequeue); + if (deq == 0) + xhci_warn(xhci, "WARN something wrong with SW event " + "ring dequeue ptr.\n"); + /* Update HC event ring dequeue pointer */ + temp_64 &= ERST_PTR_MASK; + temp_64 |= ((u64) deq & (u64) ~ERST_PTR_MASK); + } + + /* Clear the event handler busy flag (RW1C); event ring is empty. */ + temp_64 |= ERST_EHB; + xhci_write_64(xhci, temp_64, &xhci->ir_set->erst_dequeue); + + spin_unlock(&xhci->lock); + + return IRQ_HANDLED; +} + +irqreturn_t xhci_msi_irq(int irq, struct usb_hcd *hcd) +{ + irqreturn_t ret; + + set_bit(HCD_FLAG_SAW_IRQ, &hcd->flags); + + ret = xhci_irq(hcd); + + return ret; +} + /**** Endpoint Ring Operations ****/ /* @@ -1827,10 +2305,12 @@ static int prepare_transfer(struct xhci_hcd *xhci, unsigned int stream_id, unsigned int num_trbs, struct urb *urb, - struct xhci_td **td, + unsigned int td_index, gfp_t mem_flags) { int ret; + struct urb_priv *urb_priv; + struct xhci_td *td; struct xhci_ring *ep_ring; struct xhci_ep_ctx *ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); @@ -1846,24 +2326,29 @@ static int prepare_transfer(struct xhci_hcd *xhci, num_trbs, mem_flags); if (ret) return ret; - *td = kzalloc(sizeof(struct xhci_td), mem_flags); - if (!*td) - return -ENOMEM; - INIT_LIST_HEAD(&(*td)->td_list); - INIT_LIST_HEAD(&(*td)->cancelled_td_list); - ret = usb_hcd_link_urb_to_ep(xhci_to_hcd(xhci), urb); - if (unlikely(ret)) { - kfree(*td); - return ret; + urb_priv = urb->hcpriv; + td = urb_priv->td[td_index]; + + INIT_LIST_HEAD(&td->td_list); + INIT_LIST_HEAD(&td->cancelled_td_list); + + if (td_index == 0) { + ret = usb_hcd_link_urb_to_ep(xhci_to_hcd(xhci), urb); + if (unlikely(ret)) { + xhci_urb_free_priv(xhci, urb_priv); + urb->hcpriv = NULL; + return ret; + } } - (*td)->urb = urb; - urb->hcpriv = (void *) (*td); + td->urb = urb; /* Add this TD to the tail of the endpoint ring's TD list */ - list_add_tail(&(*td)->td_list, &ep_ring->td_list); - (*td)->start_seg = ep_ring->enq_seg; - (*td)->first_trb = ep_ring->enqueue; + list_add_tail(&td->td_list, &ep_ring->td_list); + td->start_seg = ep_ring->enq_seg; + td->first_trb = ep_ring->enqueue; + + urb_priv->td[td_index] = td; return 0; } @@ -1938,7 +2423,7 @@ static void giveback_first_trb(struct xhci_hcd *xhci, int slot_id, */ wmb(); start_trb->field[3] |= start_cycle; - ring_ep_doorbell(xhci, slot_id, ep_index, stream_id); + xhci_ring_ep_doorbell(xhci, slot_id, ep_index, stream_id); } /* @@ -2002,6 +2487,7 @@ static int queue_bulk_sg_tx(struct xhci_hcd *xhci, gfp_t mem_flags, { struct xhci_ring *ep_ring; unsigned int num_trbs; + struct urb_priv *urb_priv; struct xhci_td *td; struct scatterlist *sg; int num_sgs; @@ -2022,9 +2508,13 @@ static int queue_bulk_sg_tx(struct xhci_hcd *xhci, gfp_t mem_flags, trb_buff_len = prepare_transfer(xhci, xhci->devs[slot_id], ep_index, urb->stream_id, - num_trbs, urb, &td, mem_flags); + num_trbs, urb, 0, mem_flags); if (trb_buff_len < 0) return trb_buff_len; + + urb_priv = urb->hcpriv; + td = urb_priv->td[0]; + /* * Don't give the first TRB to the hardware (by toggling the cycle bit) * until we've finished creating all the other TRBs. The ring's cycle @@ -2145,6 +2635,7 @@ int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags, struct urb *urb, int slot_id, unsigned int ep_index) { struct xhci_ring *ep_ring; + struct urb_priv *urb_priv; struct xhci_td *td; int num_trbs; struct xhci_generic_trb *start_trb; @@ -2190,10 +2681,13 @@ int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags, ret = prepare_transfer(xhci, xhci->devs[slot_id], ep_index, urb->stream_id, - num_trbs, urb, &td, mem_flags); + num_trbs, urb, 0, mem_flags); if (ret < 0) return ret; + urb_priv = urb->hcpriv; + td = urb_priv->td[0]; + /* * Don't give the first TRB to the hardware (by toggling the cycle bit) * until we've finished creating all the other TRBs. The ring's cycle @@ -2279,6 +2773,7 @@ int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags, struct xhci_generic_trb *start_trb; int start_cycle; u32 field, length_field; + struct urb_priv *urb_priv; struct xhci_td *td; ep_ring = xhci_urb_to_transfer_ring(xhci, urb); @@ -2306,10 +2801,13 @@ int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags, num_trbs++; ret = prepare_transfer(xhci, xhci->devs[slot_id], ep_index, urb->stream_id, - num_trbs, urb, &td, mem_flags); + num_trbs, urb, 0, mem_flags); if (ret < 0) return ret; + urb_priv = urb->hcpriv; + td = urb_priv->td[0]; + /* * Don't give the first TRB to the hardware (by toggling the cycle bit) * until we've finished creating all the other TRBs. The ring's cycle @@ -2366,6 +2864,224 @@ int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags, return 0; } +static int count_isoc_trbs_needed(struct xhci_hcd *xhci, + struct urb *urb, int i) +{ + int num_trbs = 0; + u64 addr, td_len, running_total; + + addr = (u64) (urb->transfer_dma + urb->iso_frame_desc[i].offset); + td_len = urb->iso_frame_desc[i].length; + + running_total = TRB_MAX_BUFF_SIZE - + (addr & ((1 << TRB_MAX_BUFF_SHIFT) - 1)); + if (running_total != 0) + num_trbs++; + + while (running_total < td_len) { + num_trbs++; + running_total += TRB_MAX_BUFF_SIZE; + } + + return num_trbs; +} + +/* This is for isoc transfer */ +static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags, + struct urb *urb, int slot_id, unsigned int ep_index) +{ + struct xhci_ring *ep_ring; + struct urb_priv *urb_priv; + struct xhci_td *td; + int num_tds, trbs_per_td; + struct xhci_generic_trb *start_trb; + bool first_trb; + int start_cycle; + u32 field, length_field; + int running_total, trb_buff_len, td_len, td_remain_len, ret; + u64 start_addr, addr; + int i, j; + + ep_ring = xhci->devs[slot_id]->eps[ep_index].ring; + + num_tds = urb->number_of_packets; + if (num_tds < 1) { + xhci_dbg(xhci, "Isoc URB with zero packets?\n"); + return -EINVAL; + } + + if (!in_interrupt()) + dev_dbg(&urb->dev->dev, "ep %#x - urb len = %#x (%d)," + " addr = %#llx, num_tds = %d\n", + urb->ep->desc.bEndpointAddress, + urb->transfer_buffer_length, + urb->transfer_buffer_length, + (unsigned long long)urb->transfer_dma, + num_tds); + + start_addr = (u64) urb->transfer_dma; + start_trb = &ep_ring->enqueue->generic; + start_cycle = ep_ring->cycle_state; + + /* Queue the first TRB, even if it's zero-length */ + for (i = 0; i < num_tds; i++) { + first_trb = true; + + running_total = 0; + addr = start_addr + urb->iso_frame_desc[i].offset; + td_len = urb->iso_frame_desc[i].length; + td_remain_len = td_len; + + trbs_per_td = count_isoc_trbs_needed(xhci, urb, i); + + ret = prepare_transfer(xhci, xhci->devs[slot_id], ep_index, + urb->stream_id, trbs_per_td, urb, i, mem_flags); + if (ret < 0) + return ret; + + urb_priv = urb->hcpriv; + td = urb_priv->td[i]; + + for (j = 0; j < trbs_per_td; j++) { + u32 remainder = 0; + field = 0; + + if (first_trb) { + /* Queue the isoc TRB */ + field |= TRB_TYPE(TRB_ISOC); + /* Assume URB_ISO_ASAP is set */ + field |= TRB_SIA; + if (i > 0) + field |= ep_ring->cycle_state; + first_trb = false; + } else { + /* Queue other normal TRBs */ + field |= TRB_TYPE(TRB_NORMAL); + field |= ep_ring->cycle_state; + } + + /* Chain all the TRBs together; clear the chain bit in + * the last TRB to indicate it's the last TRB in the + * chain. + */ + if (j < trbs_per_td - 1) { + field |= TRB_CHAIN; + } else { + td->last_trb = ep_ring->enqueue; + field |= TRB_IOC; + } + + /* Calculate TRB length */ + trb_buff_len = TRB_MAX_BUFF_SIZE - + (addr & ((1 << TRB_MAX_BUFF_SHIFT) - 1)); + if (trb_buff_len > td_remain_len) + trb_buff_len = td_remain_len; + + remainder = xhci_td_remainder(td_len - running_total); + length_field = TRB_LEN(trb_buff_len) | + remainder | + TRB_INTR_TARGET(0); + queue_trb(xhci, ep_ring, false, false, + lower_32_bits(addr), + upper_32_bits(addr), + length_field, + /* We always want to know if the TRB was short, + * or we won't get an event when it completes. + * (Unless we use event data TRBs, which are a + * waste of space and HC resources.) + */ + field | TRB_ISP); + running_total += trb_buff_len; + + addr += trb_buff_len; + td_remain_len -= trb_buff_len; + } + + /* Check TD length */ + if (running_total != td_len) { + xhci_err(xhci, "ISOC TD length unmatch\n"); + return -EINVAL; + } + } + + wmb(); + start_trb->field[3] |= start_cycle; + + xhci_ring_ep_doorbell(xhci, slot_id, ep_index, urb->stream_id); + return 0; +} + +/* + * Check transfer ring to guarantee there is enough room for the urb. + * Update ISO URB start_frame and interval. + * Update interval as xhci_queue_intr_tx does. Just use xhci frame_index to + * update the urb->start_frame by now. + * Always assume URB_ISO_ASAP set, and NEVER use urb->start_frame as input. + */ +int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags, + struct urb *urb, int slot_id, unsigned int ep_index) +{ + struct xhci_virt_device *xdev; + struct xhci_ring *ep_ring; + struct xhci_ep_ctx *ep_ctx; + int start_frame; + int xhci_interval; + int ep_interval; + int num_tds, num_trbs, i; + int ret; + + xdev = xhci->devs[slot_id]; + ep_ring = xdev->eps[ep_index].ring; + ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); + + num_trbs = 0; + num_tds = urb->number_of_packets; + for (i = 0; i < num_tds; i++) + num_trbs += count_isoc_trbs_needed(xhci, urb, i); + + /* Check the ring to guarantee there is enough room for the whole urb. + * Do not insert any td of the urb to the ring if the check failed. + */ + ret = prepare_ring(xhci, ep_ring, ep_ctx->ep_info & EP_STATE_MASK, + num_trbs, mem_flags); + if (ret) + return ret; + + start_frame = xhci_readl(xhci, &xhci->run_regs->microframe_index); + start_frame &= 0x3fff; + + urb->start_frame = start_frame; + if (urb->dev->speed == USB_SPEED_LOW || + urb->dev->speed == USB_SPEED_FULL) + urb->start_frame >>= 3; + + xhci_interval = EP_INTERVAL_TO_UFRAMES(ep_ctx->ep_info); + ep_interval = urb->interval; + /* Convert to microframes */ + if (urb->dev->speed == USB_SPEED_LOW || + urb->dev->speed == USB_SPEED_FULL) + ep_interval *= 8; + /* FIXME change this to a warning and a suggestion to use the new API + * to set the polling interval (once the API is added). + */ + if (xhci_interval != ep_interval) { + if (!printk_ratelimit()) + dev_dbg(&urb->dev->dev, "Driver uses different interval" + " (%d microframe%s) than xHCI " + "(%d microframe%s)\n", + ep_interval, + ep_interval == 1 ? "" : "s", + xhci_interval, + xhci_interval == 1 ? "" : "s"); + urb->interval = xhci_interval; + /* Convert back to frames for LS/FS devices */ + if (urb->dev->speed == USB_SPEED_LOW || + urb->dev->speed == USB_SPEED_FULL) + urb->interval /= 8; + } + return xhci_queue_isoc_tx(xhci, GFP_ATOMIC, urb, slot_id, ep_index); +} + /**** Command Ring Operations ****/ /* Generic function for queueing a command TRB on the command ring. @@ -2468,15 +3184,20 @@ int xhci_queue_evaluate_context(struct xhci_hcd *xhci, dma_addr_t in_ctx_ptr, false); } +/* + * Suspend is set to indicate "Stop Endpoint Command" is being issued to stop + * activity on an endpoint that is about to be suspended. + */ int xhci_queue_stop_endpoint(struct xhci_hcd *xhci, int slot_id, - unsigned int ep_index) + unsigned int ep_index, int suspend) { u32 trb_slot_id = SLOT_ID_FOR_TRB(slot_id); u32 trb_ep_index = EP_ID_FOR_TRB(ep_index); u32 type = TRB_TYPE(TRB_STOP_RING); + u32 trb_suspend = SUSPEND_PORT_FOR_TRB(suspend); return queue_command(xhci, 0, 0, 0, - trb_slot_id | trb_ep_index | type, false); + trb_slot_id | trb_ep_index | type | trb_suspend, false); } /* Set Transfer Ring Dequeue Pointer command. |