/*
* Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
FILE_LICENCE ( GPL2_OR_LATER );
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <strings.h>
#include <errno.h>
#include <byteswap.h>
#include <ipxe/malloc.h>
#include <ipxe/umalloc.h>
#include <ipxe/pci.h>
#include <ipxe/usb.h>
#include <ipxe/profile.h>
#include "xhci.h"
/** @file
*
* USB eXtensible Host Controller Interface (xHCI) driver
*
*/
/** Message transfer profiler */
static struct profiler xhci_message_profiler __profiler =
{ .name = "xhci.message" };
/** Stream transfer profiler */
static struct profiler xhci_stream_profiler __profiler =
{ .name = "xhci.stream" };
/** Event ring profiler */
static struct profiler xhci_event_profiler __profiler =
{ .name = "xhci.event" };
/** Transfer event profiler */
static struct profiler xhci_transfer_profiler __profiler =
{ .name = "xhci.transfer" };
/* Disambiguate the various error causes */
#define EIO_DATA \
__einfo_error ( EINFO_EIO_DATA )
#define EINFO_EIO_DATA \
__einfo_uniqify ( EINFO_EIO, ( 2 - 0 ), \
"Data buffer error" )
#define EIO_BABBLE \
__einfo_error ( EINFO_EIO_BABBLE )
#define EINFO_EIO_BABBLE \
__einfo_uniqify ( EINFO_EIO, ( 3 - 0 ), \
"Babble detected" )
#define EIO_USB \
__einfo_error ( EINFO_EIO_USB )
#define EINFO_EIO_USB \
__einfo_uniqify ( EINFO_EIO, ( 4 - 0 ), \
"USB transaction error" )
#define EIO_TRB \
__einfo_error ( EINFO_EIO_TRB )
#define EINFO_EIO_TRB \
__einfo_uniqify ( EINFO_EIO, ( 5 - 0 ), \
"TRB error" )
#define EIO_STALL \
__einfo_error ( EINFO_EIO_STALL )
#define EINFO_EIO_STALL \
__einfo_uniqify ( EINFO_EIO, ( 6 - 0 ), \
"Stall error" )
#define EIO_RESOURCE \
__einfo_error ( EINFO_EIO_RESOURCE )
#define EINFO_EIO_RESOURCE \
__einfo_uniqify ( EINFO_EIO, ( 7 - 0 ), \
"Resource error" )
#define EIO_BANDWIDTH \
__einfo_error ( EINFO_EIO_BANDWIDTH )
#define EINFO_EIO_BANDWIDTH \
__einfo_uniqify ( EINFO_EIO, ( 8 - 0 ), \
"Bandwidth error" )
#define EIO_NO_SLOTS \
__einfo_error ( EINFO_EIO_NO_SLOTS )
#define EINFO_EIO_NO_SLOTS \
__einfo_uniqify ( EINFO_EIO, ( 9 - 0 ), \
"No slots available" )
#define EIO_STREAM_TYPE \
__einfo_error ( EINFO_EIO_STREAM_TYPE )
#define EINFO_EIO_STREAM_TYPE \
__einfo_uniqify ( EINFO_EIO, ( 10 - 0 ), \
"Invalid stream type" )
#define EIO_SLOT \
__einfo_error ( EINFO_EIO_SLOT )
#define EINFO_EIO_SLOT \
__einfo_uniqify ( EINFO_EIO, ( 11 - 0 ), \
"Slot not enabled" )
#define EIO_ENDPOINT \
__einfo_error ( EINFO_EIO_ENDPOINT )
#define EINFO_EIO_ENDPOINT \
__einfo_uniqify ( EINFO_EIO, ( 12 - 0 ), \
"Endpoint not enabled" )
#define EIO_SHORT \
__einfo_error ( EINFO_EIO_SHORT )
#define EINFO_EIO_SHORT \
__einfo_uniqify ( EINFO_EIO, ( 13 - 0 ), \
"Short packet" )
#define EIO_UNDERRUN \
__einfo_error ( EINFO_EIO_UNDERRUN )
#define EINFO_EIO_UNDERRUN \
__einfo_uniqify ( EINFO_EIO, ( 14 - 0 ), \
"Ring underrun" )
#define EIO_OVERRUN \
__einfo_error ( EINFO_EIO_OVERRUN )
#define EINFO_EIO_OVERRUN \
__einfo_uniqify ( EINFO_EIO, ( 15 - 0 ), \
"Ring overrun" )
#define EIO_VF_RING_FULL \
__einfo_error ( EINFO_EIO_VF_RING_FULL )
#define EINFO_EIO_VF_RING_FULL \
__einfo_uniqify ( EINFO_EIO, ( 16 - 0 ), \
"Virtual function event ring full" )
#define EIO_PARAMETER \
__einfo_error ( EINFO_EIO_PARAMETER )
#define EINFO_EIO_PARAMETER \
__einfo_uniqify ( EINFO_EIO, ( 17 - 0 ), \
"Parameter error" )
#define EIO_BANDWIDTH_OVERRUN \
__einfo_error ( EINFO_EIO_BANDWIDTH_OVERRUN )
#define EINFO_EIO_BANDWIDTH_OVERRUN \
__einfo_uniqify ( EINFO_EIO, ( 18 - 0 ), \
"Bandwidth overrun" )
#define EIO_CONTEXT \
__einfo_error ( EINFO_EIO_CONTEXT )
#define EINFO_EIO_CONTEXT \
__einfo_uniqify ( EINFO_EIO, ( 19 - 0 ), \
"Context state error" )
#define EIO_NO_PING \
__einfo_error ( EINFO_EIO_NO_PING )
#define EINFO_EIO_NO_PING \
__einfo_uniqify ( EINFO_EIO, ( 20 - 0 ), \
"No ping response" )
#define EIO_RING_FULL \
__einfo_error ( EINFO_EIO_RING_FULL )
#define EINFO_EIO_RING_FULL \
__einfo_uniqify ( EINFO_EIO, ( 21 - 0 ), \
"Event ring full" )
#define EIO_INCOMPATIBLE \
__einfo_error ( EINFO_EIO_INCOMPATIBLE )
#define EINFO_EIO_INCOMPATIBLE \
__einfo_uniqify ( EINFO_EIO, ( 22 - 0 ), \
"Incompatible device" )
#define EIO_MISSED \
__einfo_error ( EINFO_EIO_MISSED )
#define EINFO_EIO_MISSED \
__einfo_uniqify ( EINFO_EIO, ( 23 - 0 ), \
"Missed service error" )
#define EIO_CMD_STOPPED \
__einfo_error ( EINFO_EIO_CMD_STOPPED )
#define EINFO_EIO_CMD_STOPPED \
__einfo_uniqify ( EINFO_EIO, ( 24 - 0 ), \
"Command ring stopped" )
#define EIO_CMD_ABORTED \
__einfo_error ( EINFO_EIO_CMD_ABORTED )
#define EINFO_EIO_CMD_ABORTED \
__einfo_uniqify ( EINFO_EIO, ( 25 - 0 ), \
"Command aborted" )
#define EIO_STOP \
__einfo_error ( EINFO_EIO_STOP )
#define EINFO_EIO_STOP \
__einfo_uniqify ( EINFO_EIO, ( 26 - 0 ), \
"Stopped" )
#define EIO_STOP_LEN \
__einfo_error ( EINFO_EIO_STOP_LEN )
#define EINFO_EIO_STOP_LEN \
__einfo_uniqify ( EINFO_EIO, ( 27 - 0 ), \
"Stopped - length invalid" )
#define EIO_STOP_SHORT \
__einfo_error ( EINFO_EIO_STOP_SHORT )
#define EINFO_EIO_STOP_SHORT \
__einfo_uniqify ( EINFO_EIO, ( 28 - 0 ), \
"Stopped - short packet" )
#define EIO_LATENCY \
__einfo_error ( EINFO_EIO_LATENCY )
#define EINFO_EIO_LATENCY \
__einfo_uniqify ( EINFO_EIO, ( 29 - 0 ), \
"Maximum exit latency too large" )
#define EIO_ISOCH \
__einfo_error ( EINFO_EIO_ISOCH )
#define EINFO_EIO_ISOCH \
__einfo_uniqify ( EINFO_EIO, ( 31 - 0 ), \
"Isochronous buffer overrun" )
#define EPROTO_LOST \
__einfo_error ( EINFO_EPROTO_LOST )
#define EINFO_EPROTO_LOST \
__einfo_uniqify ( EINFO_EPROTO, ( 32 - 32 ), \
"Event lost" )
#define EPROTO_UNDEFINED \
__einfo_error ( EINFO_EPROTO_UNDEFINED )
#define EINFO_EPROTO_UNDEFINED \
__einfo_uniqify ( EINFO_EPROTO, ( 33 - 32 ), \
"Undefined error" )
#define EPROTO_STREAM_ID \
__einfo_error ( EINFO_EPROTO_STREAM_ID )
#define EINFO_EPROTO_STREAM_ID \
__einfo_uniqify ( EINFO_EPROTO, ( 34 - 32 ), \
"Invalid stream ID" )
#define EPROTO_SECONDARY \
__einfo_error ( EINFO_EPROTO_SECONDARY )
#define EINFO_EPROTO_SECONDARY \
__einfo_uniqify ( EINFO_EPROTO, ( 35 - 32 ), \
"Secondary bandwidth error" )
#define EPROTO_SPLIT \
__einfo_error ( EINFO_EPROTO_SPLIT )
#define EINFO_EPROTO_SPLIT \
__einfo_uniqify ( EINFO_EPROTO, ( 36 - 32 ), \
"Split transaction error" )
#define ECODE(code) \
( ( (code) < 32 ) ? \
EUNIQ ( EINFO_EIO, ( (code) & 31 ), EIO_DATA, EIO_BABBLE, \
EIO_USB, EIO_TRB, EIO_STALL, EIO_RESOURCE, \
EIO_BANDWIDTH, EIO_NO_SLOTS, EIO_STREAM_TYPE, \
EIO_SLOT, EIO_ENDPOINT, EIO_SHORT, EIO_UNDERRUN, \
EIO_OVERRUN, EIO_VF_RING_FULL, EIO_PARAMETER, \
EIO_BANDWIDTH_OVERRUN, EIO_CONTEXT, EIO_NO_PING, \
EIO_RING_FULL, EIO_INCOMPATIBLE, EIO_MISSED, \
EIO_CMD_STOPPED, EIO_CMD_ABORTED, EIO_STOP, \
EIO_STOP_LEN, EIO_STOP_SHORT, EIO_LATENCY, \
EIO_ISOCH ) : \
( (code) < 64 ) ? \
EUNIQ ( EINFO_EPROTO, ( (code) & 31 ), EPROTO_LOST, \
EPROTO_UNDEFINED, EPROTO_STREAM_ID, \
EPROTO_SECONDARY, EPROTO_SPLIT ) : \
EFAULT )
/******************************************************************************
*
* Register access
*
******************************************************************************
*/
/**
* Initialise device
*
* @v xhci xHCI device
* @v regs MMIO registers
*/
static void xhci_init ( struct xhci_device *xhci, void *regs ) {
uint32_t hcsparams1;
uint32_t hcsparams2;
uint32_t hccparams1;
uint32_t pagesize;
size_t caplength;
size_t rtsoff;
size_t dboff;
/* Locate capability, operational, runtime, and doorbell registers */
xhci->cap = regs;
caplength = readb ( xhci->cap + XHCI_CAP_CAPLENGTH );
rtsoff = readl ( xhci->cap + XHCI_CAP_RTSOFF );
dboff = readl ( xhci->cap + XHCI_CAP_DBOFF );
xhci->op = ( xhci->cap + caplength );
xhci->run = ( xhci->cap + rtsoff );
xhci->db = ( xhci->cap + dboff );
DBGC2 ( xhci, "XHCI %p cap %08lx op %08lx run %08lx db %08lx\n",
xhci, virt_to_phys ( xhci->cap ), virt_to_phys ( xhci->op ),
virt_to_phys ( xhci->run ), virt_to_phys ( xhci->db ) );
/* Read structural parameters 1 */
hcsparams1 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS1 );
xhci->slots = XHCI_HCSPARAMS1_SLOTS ( hcsparams1 );
xhci->intrs = XHCI_HCSPARAMS1_INTRS ( hcsparams1 );
xhci->ports = XHCI_HCSPARAMS1_PORTS ( hcsparams1 );
DBGC ( xhci, "XHCI %p has %d slots %d intrs %d ports\n",
xhci, xhci->slots, xhci->intrs, xhci->ports );
/* Read structural parameters 2 */
hcsparams2 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS2 );
xhci->scratchpads = XHCI_HCSPARAMS2_SCRATCHPADS ( hcsparams2 );
DBGC2 ( xhci, "XHCI %p needs %d scratchpads\n",
xhci, xhci->scratchpads );
/* Read capability parameters 1 */
hccparams1 = readl ( xhci->cap + XHCI_CAP_HCCPARAMS1 );
xhci->addr64 = XHCI_HCCPARAMS1_ADDR64 ( hccparams1 );
xhci->csz_shift = XHCI_HCCPARAMS1_CSZ_SHIFT ( hccparams1 );
xhci->xecp = XHCI_HCCPARAMS1_XECP ( hccparams1 );
/* Read page size */
pagesize = readl ( xhci->op + XHCI_OP_PAGESIZE );
xhci->pagesize = XHCI_PAGESIZE ( pagesize );
assert ( xhci->pagesize != 0 );
assert ( ( ( xhci->pagesize ) & ( xhci->pagesize - 1 ) ) == 0 );
DBGC2 ( xhci, "XHCI %p page size %zd bytes\n",
xhci, xhci->pagesize );
}
/**
* Find extended capability
*
* @v xhci xHCI device
* @v id Capability ID
* @v offset Offset to previous extended capability instance, or zero
* @ret offset Offset to extended capability, or zero if not found
*/
static unsigned int xhci_extended_capability ( struct xhci_device *xhci,
unsigned int id,
unsigned int offset ) {
uint32_t xecp;
unsigned int next;
/* Locate the extended capability */
while ( 1 ) {
/* Locate first or next capability as applicable */
if ( offset ) {
xecp = readl ( xhci->cap + offset );
next = XHCI_XECP_NEXT ( xecp );
} else {
next = xhci->xecp;
}
if ( ! next )
return 0;
offset += next;
/* Check if this is the requested capability */
xecp = readl ( xhci->cap + offset );
if ( XHCI_XECP_ID ( xecp ) == id )
return offset;
}
}
/**
* Write potentially 64-bit register
*
* @v xhci xHCI device
* @v value Value
* @v reg Register address
* @ret rc Return status code
*/
static inline __attribute__ (( always_inline )) int
xhci_writeq ( struct xhci_device *xhci, physaddr_t value, void *reg ) {
/* If this is a 32-bit build, then this can never fail
* (allowing the compiler to optimise out the error path).
*/
if ( sizeof ( value ) <= sizeof ( uint32_t ) ) {
writel ( value, reg );
writel ( 0, ( reg + sizeof ( uint32_t ) ) );
return 0;
}
/* If the device does not support 64-bit addresses and this
* address is outside the 32-bit address space, then fail.
*/
if ( ( value & ~0xffffffffULL ) && ! xhci->addr64 ) {
DBGC ( xhci, "XHCI %p cannot access address %lx\n",
xhci, value );
return -ENOTSUP;
}
/* If this is a 64-bit build, then writeq() is available */
writeq ( value, reg );
return 0;
}
/**
* Calculate buffer alignment
*
* @v len Length
* @ret align Buffer alignment
*
* Determine alignment required for a buffer which must be aligned to
* at least XHCI_MIN_ALIGN and which must not cross a page boundary.
*/
static inline size_t xhci_align ( size_t len ) {
size_t align;
/* Align to own length (rounded up to a power of two) */
align = ( 1 << fls ( len - 1 ) );
/* Round up to XHCI_MIN_ALIGN if needed */
if ( align < XHCI_MIN_ALIGN )
align = XHCI_MIN_ALIGN;
return align;
}
/**
* Calculate device context offset
*
* @v xhci xHCI device
* @v ctx Context index
*/
static inline size_t xhci_device_context_offset ( struct xhci_device *xhci,
unsigned int ctx ) {
return ( XHCI_DCI ( ctx ) << xhci->csz_shift );
}
/**
* Calculate input context offset
*
* @v xhci xHCI device
* @v ctx Context index
*/
static inline size_t xhci_input_context_offset ( struct xhci_device *xhci,
unsigned int ctx ) {
return ( XHCI_ICI ( ctx ) << xhci->csz_shift );
}
/******************************************************************************
*
* Diagnostics
*
******************************************************************************
*/
/**
* Dump host controller registers
*
* @v xhci xHCI device
*/
static inline void xhci_dump ( struct xhci_device *xhci ) {
uint32_t usbcmd;
uint32_t usbsts;
uint32_t pagesize;
uint32_t dnctrl;
uint32_t config;
/* Do nothing unless debugging is enabled */
if ( ! DBG_LOG )
return;
/* Dump USBCMD */
usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
DBGC ( xhci, "XHCI %p USBCMD %08x%s%s\n", xhci, usbcmd,
( ( usbcmd & XHCI_USBCMD_RUN ) ? " run" : "" ),
( ( usbcmd & XHCI_USBCMD_HCRST ) ? " hcrst" : "" ) );
/* Dump USBSTS */
usbsts = readl ( xhci->op + XHCI_OP_USBSTS );
DBGC ( xhci, "XHCI %p USBSTS %08x%s\n", xhci, usbsts,
( ( usbsts & XHCI_USBSTS_HCH ) ? " hch" : "" ) );
/* Dump PAGESIZE */
pagesize = readl ( xhci->op + XHCI_OP_PAGESIZE );
DBGC ( xhci, "XHCI %p PAGESIZE %08x\n", xhci, pagesize );
/* Dump DNCTRL */
dnctrl = readl ( xhci->op + XHCI_OP_DNCTRL );
DBGC ( xhci, "XHCI %p DNCTRL %08x\n", xhci, dnctrl );
/* Dump CONFIG */
config = readl ( xhci->op + XHCI_OP_CONFIG );
DBGC ( xhci, "XHCI %p CONFIG %08x\n", xhci, config );
}
/**
* Dump port registers
*
* @v xhci xHCI device
* @v port Port number
*/
static inline void xhci_dump_port ( struct xhci_device *xhci,
unsigned int port ) {
uint32_t portsc;
uint32_t portpmsc;
uint32_t portli;
uint32_t porthlpmc;
/* Do nothing unless debugging is enabled */
if ( ! DBG_LOG )
return;
/* Dump PORTSC */
portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port ) );
DBGC ( xhci, "XHCI %p port %d PORTSC %08x%s%s%s%s psiv=%d\n",
xhci, port, portsc,
( ( portsc & XHCI_PORTSC_CCS ) ? " ccs" : "" ),
( ( portsc & XHCI_PORTSC_PED ) ? " ped" : "" ),
( ( portsc & XHCI_PORTSC_PR ) ? " pr" : "" ),
( ( portsc & XHCI_PORTSC_PP ) ? " pp" : "" ),
XHCI_PORTSC_PSIV ( portsc ) );
/* Dump PORTPMSC */
portpmsc = readl ( xhci->op + XHCI_OP_PORTPMSC ( port ) );
DBGC ( xhci, "XHCI %p port %d PORTPMSC %08x\n", xhci, port, portpmsc );
/* Dump PORTLI */
portli = readl ( xhci->op + XHCI_OP_PORTLI ( port ) );
DBGC ( xhci, "XHCI %p port %d PORTLI %08x\n", xhci, port, portli );
/* Dump PORTHLPMC */
porthlpmc = readl ( xhci->op + XHCI_OP_PORTHLPMC ( port ) );
DBGC ( xhci, "XHCI %p port %d PORTHLPMC %08x\n",
xhci, port, porthlpmc );
}
/******************************************************************************
*
* USB legacy support
*
******************************************************************************
*/
/**
* Initialise USB legacy support
*
* @v xhci xHCI device
*/
static void xhci_legacy_init ( struct xhci_device *xhci ) {
unsigned int legacy;
uint8_t bios;
/* Locate USB legacy support capability (if present) */
legacy = xhci_extended_capability ( xhci, XHCI_XECP_ID_LEGACY, 0 );
if ( ! legacy ) {
/* Not an error; capability may not be present */
DBGC ( xhci, "XHCI %p has no USB legacy support capability\n",
xhci );
return;
}
/* Check if legacy USB support is enabled */
bios = readb ( xhci->cap + legacy + XHCI_USBLEGSUP_BIOS );
if ( ! ( bios & XHCI_USBLEGSUP_BIOS_OWNED ) ) {
/* Not an error; already owned by OS */
DBGC ( xhci, "XHCI %p USB legacy support already disabled\n",
xhci );
return;
}
/* Record presence of USB legacy support capability */
xhci->legacy = legacy;
}
/**
* Claim ownership from BIOS
*
* @v xhci xHCI device
* @ret rc Return status code
*/
static int xhci_legacy_claim ( struct xhci_device *xhci ) {
uint32_t ctlsts;
uint8_t bios;
unsigned int i;
/* Do nothing unless legacy support capability is present */
if ( ! xhci->legacy )
return 0;
/* Claim ownership */
writeb ( XHCI_USBLEGSUP_OS_OWNED,
xhci->cap + xhci->legacy + XHCI_USBLEGSUP_OS );
/* Wait for BIOS to release ownership */
for ( i = 0 ; i < XHCI_USBLEGSUP_MAX_WAIT_MS ; i++ ) {
/* Check if BIOS has released ownership */
bios = readb ( xhci->cap + xhci->legacy + XHCI_USBLEGSUP_BIOS );
if ( ! ( bios & XHCI_USBLEGSUP_BIOS_OWNED ) ) {
DBGC ( xhci, "XHCI %p claimed ownership from BIOS\n",
xhci );
ctlsts = readl ( xhci->cap + xhci->legacy +
XHCI_USBLEGSUP_CTLSTS );
if ( ctlsts ) {
DBGC ( xhci, "XHCI %p warning: BIOS retained "
"SMIs: %08x\n", xhci, ctlsts );
}
return 0;
}
/* Delay */
mdelay ( 1 );
}
DBGC ( xhci, "XHCI %p timed out waiting for BIOS to release "
"ownership\n", xhci );
return -ETIMEDOUT;
}
/**
* Release ownership back to BIOS
*
* @v xhci xHCI device
*/
static void xhci_legacy_release ( struct xhci_device *xhci ) {
/* Do nothing unless legacy support capability is present */
if ( ! xhci->legacy )
return;
/* Release ownership */
writeb ( 0, xhci->cap + xhci->legacy + XHCI_USBLEGSUP_OS );
DBGC ( xhci, "XHCI %p released ownership to BIOS\n", xhci );
}
/******************************************************************************
*
* Supported protocols
*
******************************************************************************
*/
/**
* Transcribe port speed (for debugging)
*
* @v psi Protocol speed ID
* @ret speed Transcribed speed
*/
static inline const char * xhci_speed_name ( uint32_t psi ) {
static const char *exponents[4] = { "", "k", "M", "G" };
static char buf[ 10 /* "xxxxxXbps" + NUL */ ];
unsigned int mantissa;
unsigned int exponent;
/* Extract mantissa and exponent */
mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
/* Transcribe speed */
snprintf ( buf, sizeof ( buf ), "%d%sbps",
mantissa, exponents[exponent] );
return buf;
}
/**
* Find supported protocol extended capability for a port
*
* @v xhci xHCI device
* @v port Port number
* @ret supported Offset to extended capability, or zero if not found
*/
static unsigned int xhci_supported_protocol ( struct xhci_device *xhci,
unsigned int port ) {
unsigned int supported = 0;
unsigned int offset;
unsigned int count;
uint32_t ports;
/* Iterate over all supported protocol structures */
while ( ( supported = xhci_extended_capability ( xhci,
XHCI_XECP_ID_SUPPORTED,
supported ) ) ) {
/* Determine port range */
ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
offset = XHCI_SUPPORTED_PORTS_OFFSET ( ports );
count = XHCI_SUPPORTED_PORTS_COUNT ( ports );
/* Check if port lies within this range */
if ( ( port - offset ) < count )
return supported;
}
DBGC ( xhci, "XHCI %p port %d has no supported protocol\n",
xhci, port );
return 0;
}
/**
* Find port protocol
*
* @v xhci xHCI device
* @v port Port number
* @ret protocol USB protocol, or zero if not found
*/
static unsigned int xhci_port_protocol ( struct xhci_device *xhci,
unsigned int port ) {
unsigned int supported = xhci_supported_protocol ( xhci, port );
union {
uint32_t raw;
char text[5];
} name;
unsigned int protocol;
unsigned int type;
unsigned int psic;
unsigned int psiv;
unsigned int i;
uint32_t revision;
uint32_t ports;
uint32_t slot;
uint32_t psi;
/* Fail if there is no supported protocol */
if ( ! supported )
return 0;
/* Determine protocol version */
revision = readl ( xhci->cap + supported + XHCI_SUPPORTED_REVISION );
protocol = XHCI_SUPPORTED_REVISION_VER ( revision );
/* Describe port protocol */
if ( DBG_EXTRA ) {
name.raw = cpu_to_le32 ( readl ( xhci->cap + supported +
XHCI_SUPPORTED_NAME ) );
name.text[4] = '\0';
slot = readl ( xhci->cap + supported + XHCI_SUPPORTED_SLOT );
type = XHCI_SUPPORTED_SLOT_TYPE ( slot );
DBGC2 ( xhci, "XHCI %p port %d %sv%04x type %d",
xhci, port, name.text, protocol, type );
ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
if ( psic ) {
DBGC2 ( xhci, " speeds" );
for ( i = 0 ; i < psic ; i++ ) {
psi = readl ( xhci->cap + supported +
XHCI_SUPPORTED_PSI ( i ) );
psiv = XHCI_SUPPORTED_PSI_VALUE ( psi );
DBGC2 ( xhci, " %d:%s", psiv,
xhci_speed_name ( psi ) );
}
}
DBGC2 ( xhci, "\n" );
}
return protocol;
}
/**
* Find port slot type
*
* @v xhci xHCI device
* @v port Port number
* @ret type Slot type, or negative error
*/
static int xhci_port_slot_type ( struct xhci_device *xhci, unsigned int port ) {
unsigned int supported = xhci_supported_protocol ( xhci, port );
unsigned int type;
uint32_t slot;
/* Fail if there is no supported protocol */
if ( ! supported )
return -ENOTSUP;
/* Get slot type */
slot = readl ( xhci->cap + supported + XHCI_SUPPORTED_SLOT );
type = XHCI_SUPPORTED_SLOT_TYPE ( slot );
return type;
}
/**
* Find port speed
*
* @v xhci xHCI device
* @v port Port number
* @v psiv Protocol speed ID value
* @ret speed Port speed, or negative error
*/
static int xhci_port_speed ( struct xhci_device *xhci, unsigned int port,
unsigned int psiv ) {
unsigned int supported = xhci_supported_protocol ( xhci, port );
unsigned int psic;
unsigned int mantissa;
unsigned int exponent;
unsigned int speed;
unsigned int i;
uint32_t ports;
uint32_t psi;
/* Fail if there is no supported protocol */
if ( ! supported )
return -ENOTSUP;
/* Get protocol speed ID count */
ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
/* Use the default mappings if applicable */
if ( ! psic ) {
switch ( psiv ) {
case XHCI_SPEED_LOW : return USB_SPEED_LOW;
case XHCI_SPEED_FULL : return USB_SPEED_FULL;
case XHCI_SPEED_HIGH : return USB_SPEED_HIGH;
case XHCI_SPEED_SUPER : return USB_SPEED_SUPER;
default:
DBGC ( xhci, "XHCI %p port %d non-standard PSI value "
"%d\n", xhci, port, psiv );
return -ENOTSUP;
}
}
/* Iterate over PSI dwords looking for a match */
for ( i = 0 ; i < psic ; i++ ) {
psi = readl ( xhci->cap + supported + XHCI_SUPPORTED_PSI ( i ));
if ( psiv == XHCI_SUPPORTED_PSI_VALUE ( psi ) ) {
mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
speed = USB_SPEED ( mantissa, exponent );
return speed;
}
}
DBGC ( xhci, "XHCI %p port %d spurious PSI value %d\n",
xhci, port, psiv );
return -ENOENT;
}
/**
* Find protocol speed ID value
*
* @v xhci xHCI device
* @v port Port number
* @v speed USB speed
* @ret psiv Protocol speed ID value, or negative error
*/
static int xhci_port_psiv ( struct xhci_device *xhci, unsigned int port,
unsigned int speed ) {
unsigned int supported = xhci_supported_protocol ( xhci, port );
unsigned int psic;
unsigned int mantissa;
unsigned int exponent;
unsigned int psiv;
unsigned int i;
uint32_t ports;
uint32_t psi;
/* Fail if there is no supported protocol */
if ( ! supported )
return -ENOTSUP;
/* Get protocol speed ID count */
ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
/* Use the default mappings if applicable */
if ( ! psic ) {
switch ( speed ) {
case USB_SPEED_LOW : return XHCI_SPEED_LOW;
case USB_SPEED_FULL : return XHCI_SPEED_FULL;
case USB_SPEED_HIGH : return XHCI_SPEED_HIGH;
case USB_SPEED_SUPER : return XHCI_SPEED_SUPER;
default:
DBGC ( xhci, "XHCI %p port %d non-standad speed %d\n",
xhci, port, speed );
return -ENOTSUP;
}
}
/* Iterate over PSI dwords looking for a match */
for ( i = 0 ; i < psic ; i++ ) {
psi = readl ( xhci->cap + supported + XHCI_SUPPORTED_PSI ( i ));
mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
if ( speed == USB_SPEED ( mantissa, exponent ) ) {
psiv = XHCI_SUPPORTED_PSI_VALUE ( psi );
return psiv;
}
}
DBGC ( xhci, "XHCI %p port %d unrepresentable speed %#x\n",
xhci, port, speed );
return -ENOENT;
}
/******************************************************************************
*
* Device context base address array
*
******************************************************************************
*/
/**
* Allocate device context base address array
*
* @v xhci xHCI device
* @ret rc Return status code
*/
static int xhci_dcbaa_alloc ( struct xhci_device *xhci ) {
size_t len;
physaddr_t dcbaap;
int rc;
/* Allocate and initialise structure. Must be at least
* 64-byte aligned and must not cross a page boundary, so
* align on its own size (rounded up to a power of two and
* with a minimum of 64 bytes).
*/
len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa[0] ) );
xhci->dcbaa = malloc_dma ( len, xhci_align ( len ) );
if ( ! xhci->dcbaa ) {
DBGC ( xhci, "XHCI %p could not allocate DCBAA\n", xhci );
rc = -ENOMEM;
goto err_alloc;
}
memset ( xhci->dcbaa, 0, len );
/* Program DCBAA pointer */
dcbaap = virt_to_phys ( xhci->dcbaa );
if ( ( rc = xhci_writeq ( xhci, dcbaap,
xhci->op + XHCI_OP_DCBAAP ) ) != 0 )
goto err_writeq;
DBGC2 ( xhci, "XHCI %p DCBAA at [%08lx,%08lx)\n",
xhci, dcbaap, ( dcbaap + len ) );
return 0;
err_writeq:
free_dma ( xhci->dcbaa, len );
err_alloc:
return rc;
}
/**
* Free device context base address array
*
* @v xhci xHCI device
*/
static void xhci_dcbaa_free ( struct xhci_device *xhci ) {
size_t len;
unsigned int i;
/* Sanity check */
for ( i = 0 ; i <= xhci->slots ; i++ )
assert ( xhci->dcbaa[i] == 0 );
/* Clear DCBAA pointer */
xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_DCBAAP );
/* Free DCBAA */
len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa[0] ) );
free_dma ( xhci->dcbaa, len );
}
/******************************************************************************
*
* Scratchpad buffers
*
******************************************************************************
*/
/**
* Allocate scratchpad buffers
*
* @v xhci xHCI device
* @ret rc Return status code
*/
static int xhci_scratchpad_alloc ( struct xhci_device *xhci ) {
size_t array_len;
size_t len;
physaddr_t phys;
unsigned int i;
int rc;
/* Do nothing if no scratchpad buffers are used */
if ( ! xhci->scratchpads )
return 0;
/* Allocate scratchpads */
len = ( xhci->scratchpads * xhci->pagesize );
xhci->scratchpad = umalloc ( len );
if ( ! xhci->scratchpad ) {
DBGC ( xhci, "XHCI %p could not allocate scratchpad buffers\n",
xhci );
rc = -ENOMEM;
goto err_alloc;
}
memset_user ( xhci->scratchpad, 0, 0, len );
/* Allocate scratchpad array */
array_len = ( xhci->scratchpads * sizeof ( xhci->scratchpad_array[0] ));
xhci->scratchpad_array =
malloc_dma ( array_len, xhci_align ( array_len ) );
if ( ! xhci->scratchpad_array ) {
DBGC ( xhci, "XHCI %p could not allocate scratchpad buffer "
"array\n", xhci );
rc = -ENOMEM;
goto err_alloc_array;
}
/* Populate scratchpad array */
for ( i = 0 ; i < xhci->scratchpads ; i++ ) {
phys = user_to_phys ( xhci->scratchpad, ( i * xhci->pagesize ));
xhci->scratchpad_array[i] = phys;
}
/* Set scratchpad array pointer */
assert ( xhci->dcbaa != NULL );
xhci->dcbaa[0] = cpu_to_le64 ( virt_to_phys ( xhci->scratchpad_array ));
DBGC2 ( xhci, "XHCI %p scratchpad [%08lx,%08lx) array [%08lx,%08lx)\n",
xhci, user_to_phys ( xhci->scratchpad, 0 ),
user_to_phys ( xhci->scratchpad, len ),
virt_to_phys ( xhci->scratchpad_array ),
( virt_to_phys ( xhci->scratchpad_array ) + array_len ) );
return 0;
free_dma ( xhci->scratchpad_array, array_len );
err_alloc_array:
ufree ( xhci->scratchpad );
err_alloc:
return rc;
}
/**
* Free scratchpad buffers
*
* @v xhci xHCI device
*/
static void xhci_scratchpad_free ( struct xhci_device *xhci ) {
size_t array_len;
/* Do nothing if no scratchpad buffers are used */
if ( ! xhci->scratchpads )
return;
/* Clear scratchpad array pointer */
assert ( xhci->dcbaa != NULL );
xhci->dcbaa[0] = 0;
/* Free scratchpad array */
array_len = ( xhci->scratchpads * sizeof ( xhci->scratchpad_array[0] ));
free_dma ( xhci->scratchpad_array, array_len );
/* Free scratchpads */
ufree ( xhci->scratchpad );
}
/******************************************************************************
*
* Run / stop / reset
*
******************************************************************************
*/
/**
* Start xHCI device
*
* @v xhci xHCI device
*/
static void xhci_run ( struct xhci_device *xhci ) {
uint32_t config;
uint32_t usbcmd;
/* Configure number of device slots */
config = readl ( xhci->op + XHCI_OP_CONFIG );
config &= ~XHCI_CONFIG_MAX_SLOTS_EN_MASK;
config |= XHCI_CONFIG_MAX_SLOTS_EN ( xhci->slots );
writel ( config, xhci->op + XHCI_OP_CONFIG );
/* Set run/stop bit */
usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
usbcmd |= XHCI_USBCMD_RUN;
writel ( usbcmd, xhci->op + XHCI_OP_USBCMD );
}
/**
* Stop xHCI device
*
* @v xhci xHCI device
* @ret rc Return status code
*/
static int xhci_stop ( struct xhci_device *xhci ) {
uint32_t usbcmd;
uint32_t usbsts;
unsigned int i;
/* Clear run/stop bit */
usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
usbcmd &= ~XHCI_USBCMD_RUN;
writel ( usbcmd, xhci->op + XHCI_OP_USBCMD );
/* Wait for device to stop */
for ( i = 0 ; i < XHCI_STOP_MAX_WAIT_MS ; i++ ) {
/* Check if device is stopped */
usbsts = readl ( xhci->op + XHCI_OP_USBSTS );
if ( usbsts & XHCI_USBSTS_HCH )
return 0;
/* Delay */
mdelay ( 1 );
}
DBGC ( xhci, "XHCI %p timed out waiting for stop\n", xhci );
return -ETIMEDOUT;
}
/**
* Reset xHCI device
*
* @v xhci xHCI device
* @ret rc Return status code
*/
static int xhci_reset ( struct xhci_device *xhci ) {
uint32_t usbcmd;
unsigned int i;
int rc;
/* The xHCI specification states that resetting a running
* device may result in undefined behaviour, so try stopping
* it first.
*/
if ( ( rc = xhci_stop ( xhci ) ) != 0 ) {
/* Ignore errors and attempt to reset the device anyway */
}
/* Reset device */
writel ( XHCI_USBCMD_HCRST, xhci->op + XHCI_OP_USBCMD );
/* Wait for reset to complete */
for ( i = 0 ; i < XHCI_RESET_MAX_WAIT_MS ; i++ ) {
/* Check if reset is complete */
usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
if ( ! ( usbcmd & XHCI_USBCMD_HCRST ) )
return 0;
/* Delay */
mdelay ( 1 );
}
DBGC ( xhci, "XHCI %p timed out waiting for reset\n", xhci );
return -ETIMEDOUT;
}
/******************************************************************************
*
* Transfer request blocks
*
******************************************************************************
*/
/**
* Allocate transfer request block ring
*
* @v xhci xHCI device
* @v ring TRB ring
* @v shift Ring size (log2)
* @v slot Device slot
* @v target Doorbell target
* @v stream Doorbell stream ID
* @ret rc Return status code
*/
static int xhci_ring_alloc ( struct xhci_device *xhci,
struct xhci_trb_ring *ring,
unsigned int shift, unsigned int slot,
unsigned int target, unsigned int stream ) {
struct xhci_trb_link *link;
unsigned int count;
int rc;
/* Sanity check */
assert ( shift > 0 );
/* Initialise structure */
memset ( ring, 0, sizeof ( *ring ) );
ring->shift = shift;
count = ( 1U << shift );
ring->mask = ( count - 1 );
ring->len = ( ( count + 1 /* Link TRB */ ) * sizeof ( ring->trb[0] ) );
ring->db = ( xhci->db + ( slot * sizeof ( ring->dbval ) ) );
ring->dbval = XHCI_DBVAL ( target, stream );
/* Allocate I/O buffers */
ring->iobuf = zalloc ( count * sizeof ( ring->iobuf[0] ) );
if ( ! ring->iobuf ) {
rc = -ENOMEM;
goto err_alloc_iobuf;
}
/* Allocate TRBs */
ring->trb = malloc_dma ( ring->len, xhci_align ( ring->len ) );
if ( ! ring->trb ) {
rc = -ENOMEM;
goto err_alloc_trb;
}
memset ( ring->trb, 0, ring->len );
/* Initialise Link TRB */
link = &ring->trb[count].link;
link->next = cpu_to_le64 ( virt_to_phys ( ring->trb ) );
link->flags = XHCI_TRB_TC;
link->type = XHCI_TRB_LINK;
ring->link = link;
return 0;
free_dma ( ring->trb, ring->len );
err_alloc_trb:
free ( ring->iobuf );
err_alloc_iobuf:
return rc;
}
/**
* Free transfer request block ring
*
* @v ring TRB ring
*/
static void xhci_ring_free ( struct xhci_trb_ring *ring ) {
unsigned int count = ( 1U << ring->shift );
unsigned int i;
/* Sanity checks */
assert ( ring->cons == ring->prod );
for ( i = 0 ; i < count ; i++ )
assert ( ring->iobuf[i] == NULL );
/* Free TRBs */
free_dma ( ring->trb, ring->len );
/* Free I/O buffers */
free ( ring->iobuf );
}
/**
* Enqueue a transfer request block
*
* @v ring TRB ring
* @v iobuf I/O buffer (if any)
* @v trb Transfer request block (with empty Cycle flag)
* @ret rc Return status code
*
* This operation does not implicitly ring the doorbell register.
*/
static int xhci_enqueue ( struct xhci_trb_ring *ring, struct io_buffer *iobuf,
const union xhci_trb *trb ) {
union xhci_trb *dest;
unsigned int prod;
unsigned int mask;
unsigned int index;
unsigned int cycle;
/* Sanity check */
assert ( ! ( trb->common.flags & XHCI_TRB_C ) );
/* Fail if ring is full */
if ( ! xhci_ring_remaining ( ring ) )
return -ENOBUFS;
/* Update producer counter (and link TRB, if applicable) */
prod = ring->prod++;
mask = ring->mask;
cycle = ( ( ~( prod >> ring->shift ) ) & XHCI_TRB_C );
index = ( prod & mask );
if ( index == 0 )
ring->link->flags = ( XHCI_TRB_TC | ( cycle ^ XHCI_TRB_C ) );
/* Record I/O buffer */
ring->iobuf[index] = iobuf;
/* Enqueue TRB */
dest = &ring->trb[index];
dest->template.parameter = trb->template.parameter;
dest->template.status = trb->template.status;
wmb();
dest->template.control = ( trb->template.control |
cpu_to_le32 ( cycle ) );
return 0;
}
/**
* Dequeue a transfer request block
*
* @v ring TRB ring
* @ret iobuf I/O buffer
*/
static struct io_buffer * xhci_dequeue ( struct xhci_trb_ring *ring ) {
struct io_buffer *iobuf;
unsigned int cons;
unsigned int mask;
unsigned int index;
/* Sanity check */
assert ( xhci_ring_fill ( ring ) != 0 );
/* Update consumer counter */
cons = ring->cons++;
mask = ring->mask;
index = ( cons & mask );
/* Retrieve I/O buffer */
iobuf = ring->iobuf[index];
ring->iobuf[index] = NULL;
return iobuf;
}
/**
* Enqueue multiple transfer request blocks
*
* @v ring TRB ring
* @v iobuf I/O buffer
* @v trbs Transfer request blocks (with empty Cycle flag)
* @v count Number of transfer request blocks
* @ret rc Return status code
*
* This operation does not implicitly ring the doorbell register.
*/
static int xhci_enqueue_multi ( struct xhci_trb_ring *ring,
struct io_buffer *iobuf,
const union xhci_trb *trbs,
unsigned int count ) {
const union xhci_trb *trb = trbs;
int rc;
/* Sanity check */
assert ( iobuf != NULL );
/* Fail if ring does not have sufficient space */
if ( xhci_ring_remaining ( ring ) < count )
return -ENOBUFS;
/* Enqueue each TRB, recording the I/O buffer with the final TRB */
while ( count-- ) {
rc = xhci_enqueue ( ring, ( count ? NULL : iobuf ), trb++ );
assert ( rc == 0 ); /* Should never be able to fail */
}
return 0;
}
/**
* Dequeue multiple transfer request blocks
*
* @v ring TRB ring
* @ret iobuf I/O buffer
*/
static struct io_buffer * xhci_dequeue_multi ( struct xhci_trb_ring *ring ) {
struct io_buffer *iobuf;
/* Dequeue TRBs until we reach the final TRB for an I/O buffer */
do {
iobuf = xhci_dequeue ( ring );
} while ( iobuf == NULL );
return iobuf;
}
/**
* Ring doorbell register
*
* @v ring TRB ring
*/
static inline __attribute__ (( always_inline )) void
xhci_doorbell ( struct xhci_trb_ring *ring ) {
wmb();
writel ( ring->dbval, ring->db );
}
/******************************************************************************
*
* Command and event rings
*
******************************************************************************
*/
/**
* Allocate command ring
*
* @v xhci xHCI device
* @ret rc Return status code
*/
static int xhci_command_alloc ( struct xhci_device *xhci ) {
physaddr_t crp;
int rc;
/* Allocate TRB ring */
if ( ( rc = xhci_ring_alloc ( xhci, &xhci->command, XHCI_CMD_TRBS_LOG2,
0, 0, 0 ) ) != 0 )
goto err_ring_alloc;
/* Program command ring control register */
crp = virt_to_phys ( xhci->command.trb );
if ( ( rc = xhci_writeq ( xhci, ( crp | XHCI_CRCR_RCS ),
xhci->op + XHCI_OP_CRCR ) ) != 0 )
goto err_writeq;
DBGC2 ( xhci, "XHCI %p CRCR at [%08lx,%08lx)\n",
xhci, crp, ( crp + xhci->command.len ) );
return 0;
err_writeq:
xhci_ring_free ( &xhci->command );
err_ring_alloc:
return rc;
}
/**
* Free command ring
*
* @v xhci xHCI device
*/
static void xhci_command_free ( struct xhci_device *xhci ) {
/* Sanity check */
assert ( ( readl ( xhci->op + XHCI_OP_CRCR ) & XHCI_CRCR_CRR ) == 0 );
/* Clear command ring control register */
xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_CRCR );
/* Free TRB ring */
xhci_ring_free ( &xhci->command );
}
/**
* Allocate event ring
*
* @v xhci xHCI device
* @ret rc Return status code
*/
static int xhci_event_alloc ( struct xhci_device *xhci ) {
struct xhci_event_ring *event = &xhci->event;
unsigned int count;
size_t len;
int rc;
/* Allocate event ring */
count = ( 1 << XHCI_EVENT_TRBS_LOG2 );
len = ( count * sizeof ( event->trb[0] ) );
event->trb = malloc_dma ( len, xhci_align ( len ) );
if ( ! event->trb ) {
rc = -ENOMEM;
goto err_alloc_trb;
}
memset ( event->trb, 0, len );
/* Allocate event ring segment table */
event->segment = malloc_dma ( sizeof ( event->segment[0] ),
xhci_align ( sizeof (event->segment[0])));
if ( ! event->segment ) {
rc = -ENOMEM;
goto err_alloc_segment;
}
memset ( event->segment, 0, sizeof ( event->segment[0] ) );
event->segment[0].base = cpu_to_le64 ( virt_to_phys ( event->trb ) );
event->segment[0].count = cpu_to_le32 ( count );
/* Program event ring registers */
writel ( 1, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) );
if ( ( rc = xhci_writeq ( xhci, virt_to_phys ( event->trb ),
xhci->run + XHCI_RUN_ERDP ( 0 ) ) ) != 0 )
goto err_writeq_erdp;
if ( ( rc = xhci_writeq ( xhci, virt_to_phys ( event->segment ),
xhci->run + XHCI_RUN_ERSTBA ( 0 ) ) ) != 0 )
goto err_writeq_erstba;
DBGC2 ( xhci, "XHCI %p event ring [%08lx,%08lx) table [%08lx,%08lx)\n",
xhci, virt_to_phys ( event->trb ),
( virt_to_phys ( event->trb ) + len ),
virt_to_phys ( event->segment ),
( virt_to_phys ( event->segment ) +
sizeof (event->segment[0] ) ) );
return 0;
xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) );
err_writeq_erstba:
xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) );
err_writeq_erdp:
free_dma ( event->trb, len );
err_alloc_segment:
free_dma ( event->segment, sizeof ( event->segment[0] ) );
err_alloc_trb:
return rc;
}
/**
* Free event ring
*
* @v xhci xHCI device
*/
static void xhci_event_free ( struct xhci_device *xhci ) {
struct xhci_event_ring *event = &xhci->event;
unsigned int count;
size_t len;
/* Clear event ring registers */
writel ( 0, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) );
xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) );
xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) );
/* Free event ring segment table */
free_dma ( event->segment, sizeof ( event->segment[0] ) );
/* Free event ring */
count = ( 1 << XHCI_EVENT_TRBS_LOG2 );
len = ( count * sizeof ( event->trb[0] ) );
free_dma ( event->trb, len );
}
/**
* Handle transfer event
*
* @v xhci xHCI device
* @v transfer Transfer event TRB
*/
static void xhci_transfer ( struct xhci_device *xhci,
struct xhci_trb_transfer *transfer ) {
struct xhci_slot *slot;
struct xhci_endpoint *endpoint;
struct io_buffer *iobuf;
int rc;
/* Profile transfer events */
profile_start ( &xhci_transfer_profiler );
/* Identify slot */
if ( ( transfer->slot > xhci->slots ) ||
( ( slot = xhci->slot[transfer->slot] ) == NULL ) ) {
DBGC ( xhci, "XHCI %p transfer event invalid slot %d:\n",
xhci, transfer->slot );
DBGC_HDA ( xhci, 0, transfer, sizeof ( *transfer ) );
return;
}
/* Identify endpoint */
if ( ( transfer->endpoint > XHCI_CTX_END ) ||
( ( endpoint = slot->endpoint[transfer->endpoint] ) == NULL ) ) {
DBGC ( xhci, "XHCI %p slot %d transfer event invalid epid "
"%d:\n", xhci, slot->id, transfer->endpoint );
DBGC_HDA ( xhci, 0, transfer, sizeof ( *transfer ) );
return;
}
/* Dequeue TRB(s) */
iobuf = xhci_dequeue_multi ( &endpoint->ring );
assert ( iobuf != NULL );
/* Check for errors */
if ( ! ( ( transfer->code == XHCI_CMPLT_SUCCESS ) ||
( transfer->code == XHCI_CMPLT_SHORT ) ) ) {
/* Construct error */
rc = -ECODE ( transfer->code );
DBGC ( xhci, "XHCI %p slot %d ctx %d failed (code %d): %s\n",
xhci, slot->id, endpoint->ctx, transfer->code,
strerror ( rc ) );
DBGC_HDA ( xhci, 0, transfer, sizeof ( *transfer ) );
/* Sanity check */
assert ( ( endpoint->context->state & XHCI_ENDPOINT_STATE_MASK )
!= XHCI_ENDPOINT_RUNNING );
/* Report failure to USB core */
usb_complete_err ( endpoint->ep, iobuf, rc );
return;
}
/* Record actual transfer size */
iob_unput ( iobuf, le16_to_cpu ( transfer->residual ) );
/* Sanity check (for successful completions only) */
assert ( xhci_ring_consumed ( &endpoint->ring ) ==
le64_to_cpu ( transfer->transfer ) );
/* Report completion to USB core */
usb_complete ( endpoint->ep, iobuf );
profile_stop ( &xhci_transfer_profiler );
}
/**
* Handle command completion event
*
* @v xhci xHCI device
* @v complete Command completion event
*/
static void xhci_complete ( struct xhci_device *xhci,
struct xhci_trb_complete *complete ) {
/* Dequeue command TRB */
xhci_dequeue ( &xhci->command );
/* Sanity check */
assert ( xhci_ring_consumed ( &xhci->command ) ==
le64_to_cpu ( complete->command ) );
/* Record completion if applicable */
if ( xhci->completion ) {
memcpy ( xhci->completion, complete,
sizeof ( *xhci->completion ) );
xhci->completion = NULL;
} else {
DBGC ( xhci, "XHCI %p unexpected completion:\n", xhci );
DBGC_HDA ( xhci, 0, complete, sizeof ( *complete ) );
}
}
/**
* Handle port status event
*
* @v xhci xHCI device
* @v port Port status event
*/
static void xhci_port_status ( struct xhci_device *xhci,
struct xhci_trb_port_status *port ) {
uint32_t portsc;
/* Sanity check */
assert ( ( port->port > 0 ) && ( port->port <= xhci->ports ) );
/* Clear port status change bits */
portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->port ) );
portsc &= ( XHCI_PORTSC_PRESERVE | XHCI_PORTSC_CHANGE );
writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->port ) );
/* Report port status change */
usb_port_changed ( usb_port ( xhci->bus->hub, port->port ) );
}
/**
* Handle host controller event
*
* @v xhci xHCI device
* @v host Host controller event
*/
static void xhci_host_controller ( struct xhci_device *xhci,
struct xhci_trb_host_controller *host ) {
int rc;
/* Construct error */
rc = -ECODE ( host->code );
DBGC ( xhci, "XHCI %p host controller event (code %d): %s\n",
xhci, host->code, strerror ( rc ) );
}
/**
* Poll event ring
*
* @v xhci xHCI device
*/
static void xhci_event_poll ( struct xhci_device *xhci ) {
struct xhci_event_ring *event = &xhci->event;
union xhci_trb *trb;
unsigned int shift = XHCI_EVENT_TRBS_LOG2;
unsigned int count = ( 1 << shift );
unsigned int mask = ( count - 1 );
unsigned int consumed;
unsigned int type;
/* Poll for events */
profile_start ( &xhci_event_profiler );
for ( consumed = 0 ; ; consumed++ ) {
/* Stop if we reach an empty TRB */
rmb();
trb = &event->trb[ event->cons & mask ];
if ( ! ( ( trb->common.flags ^
( event->cons >> shift ) ) & XHCI_TRB_C ) )
break;
/* Handle TRB */
type = ( trb->common.type & XHCI_TRB_TYPE_MASK );
switch ( type ) {
case XHCI_TRB_TRANSFER :
xhci_transfer ( xhci, &trb->transfer );
break;
case XHCI_TRB_COMPLETE :
xhci_complete ( xhci, &trb->complete );
break;
case XHCI_TRB_PORT_STATUS:
xhci_port_status ( xhci, &trb->port );
break;
case XHCI_TRB_HOST_CONTROLLER:
xhci_host_controller ( xhci, &trb->host );
break;
default:
DBGC ( xhci, "XHCI %p unrecognised event %#x\n:",
xhci, event->cons );
DBGC_HDA ( xhci, virt_to_phys ( trb ),
trb, sizeof ( *trb ) );
break;
}
/* Consume this TRB */
event->cons++;
}
/* Update dequeue pointer if applicable */
if ( consumed ) {
xhci_writeq ( xhci, virt_to_phys ( trb ),
xhci->run + XHCI_RUN_ERDP ( 0 ) );
profile_stop ( &xhci_event_profiler );
}
}
/**
* Issue command and wait for completion
*
* @v xhci xHCI device
* @v trb Transfer request block (with empty Cycle flag)
* @ret rc Return status code
*
* On a successful completion, the TRB will be overwritten with the
* completion.
*/
static int xhci_command ( struct xhci_device *xhci, union xhci_trb *trb ) {
struct xhci_trb_complete *complete = &trb->complete;
unsigned int i;
int rc;
/* Record the completion buffer */
xhci->completion = trb;
/* Enqueue the command */
if ( ( rc = xhci_enqueue ( &xhci->command, NULL, trb ) ) != 0 )
goto err_enqueue;
/* Ring the command doorbell */
xhci_doorbell ( &xhci->command );
/* Wait for the command to complete */
for ( i = 0 ; i < XHCI_COMMAND_MAX_WAIT_MS ; i++ ) {
/* Poll event ring */
xhci_event_poll ( xhci );
/* Check for completion */
if ( ! xhci->completion ) {
if ( complete->code != XHCI_CMPLT_SUCCESS ) {
rc = -ECODE ( complete->code );
DBGC ( xhci, "XHCI %p command failed (code "
"%d): %s\n", xhci, complete->code,
strerror ( rc ) );
DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
return rc;
}
return 0;
}
/* Delay */
mdelay ( 1 );
}
/* Timeout */
DBGC ( xhci, "XHCI %p timed out waiting for completion\n", xhci );
rc = -ETIMEDOUT;
err_enqueue:
xhci->completion = NULL;
return rc;
}
/**
* Issue NOP and wait for completion
*
* @v xhci xHCI device
* @ret rc Return status code
*/
static inline int xhci_nop ( struct xhci_device *xhci ) {
union xhci_trb trb;
struct xhci_trb_common *nop = &trb.common;
int rc;
/* Construct command */
memset ( nop, 0, sizeof ( *nop ) );
nop->flags = XHCI_TRB_IOC;
nop->type = XHCI_TRB_NOP_CMD;
/* Issue command and wait for completion */
if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 )
return rc;
return 0;
}
/**
* Enable slot
*
* @v xhci xHCI device
* @v type Slot type
* @ret slot Device slot ID, or negative error
*/
static inline int xhci_enable_slot ( struct xhci_device *xhci,
unsigned int type ) {
union xhci_trb trb;
struct xhci_trb_enable_slot *enable = &trb.enable;
struct xhci_trb_complete *enabled = &trb.complete;
unsigned int slot;
int rc;
/* Construct command */
memset ( enable, 0, sizeof ( *enable ) );
enable->slot = type;
enable->type = XHCI_TRB_ENABLE_SLOT;
/* Issue command and wait for completion */
if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
DBGC ( xhci, "XHCI %p could not enable new slot: %s\n",
xhci, strerror ( rc ) );
return rc;
}
/* Extract slot number */
slot = enabled->slot;
DBGC2 ( xhci, "XHCI %p slot %d enabled\n", xhci, slot );
return slot;
}
/**
* Disable slot
*
* @v xhci xHCI device
* @v slot Device slot
* @ret rc Return status code
*/
static inline int xhci_disable_slot ( struct xhci_device *xhci,
unsigned int slot ) {
union xhci_trb trb;
struct xhci_trb_disable_slot *disable = &trb.disable;
int rc;
/* Construct command */
memset ( disable, 0, sizeof ( *disable ) );
disable->type = XHCI_TRB_DISABLE_SLOT;
disable->slot = slot;
/* Issue command and wait for completion */
if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
DBGC ( xhci, "XHCI %p could not disable slot %d: %s\n",
xhci, slot, strerror ( rc ) );
return rc;
}
DBGC2 ( xhci, "XHCI %p slot %d disabled\n", xhci, slot );
return 0;
}
/**
* Issue context-based command and wait for completion
*
* @v xhci xHCI device
* @v slot Device slot
* @v endpoint Endpoint
* @v type TRB type
* @v populate Input context populater
* @ret rc Return status code
*/
static int xhci_context ( struct xhci_device *xhci, struct xhci_slot *slot,
struct xhci_endpoint *endpoint, unsigned int type,
void ( * populate ) ( struct xhci_device *xhci,
struct xhci_slot *slot,
struct xhci_endpoint *endpoint,
void *input ) ) {
union xhci_trb trb;
struct xhci_trb_context *context = &trb.context;
size_t len;
void *input;
int rc;
/* Allocate an input context */
len = xhci_input_context_offset ( xhci, XHCI_CTX_END );
input = malloc_dma ( len, xhci_align ( len ) );
if ( ! input ) {
rc = -ENOMEM;
goto err_alloc;
}
memset ( input, 0, len );
/* Populate input context */
populate ( xhci, slot, endpoint, input );
/* Construct command */
memset ( context, 0, sizeof ( *context ) );
context->type = type;
context->input = cpu_to_le64 ( virt_to_phys ( input ) );
context->slot = slot->id;
/* Issue command and wait for completion */
if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 )
goto err_command;
err_command:
free_dma ( input, len );
err_alloc:
return rc;
}
/**
* Populate address device input context
*
* @v xhci xHCI device
* @v slot Device slot
* @v endpoint Endpoint
* @v input Input context
*/
static void xhci_address_device_input ( struct xhci_device *xhci,
struct xhci_slot *slot,
struct xhci_endpoint *endpoint,
void *input ) {
struct xhci_control_context *control_ctx;
struct xhci_slot_context *slot_ctx;
struct xhci_endpoint_context *ep_ctx;
/* Sanity checks */
assert ( endpoint->ctx == XHCI_CTX_EP0 );
/* Populate control context */
control_ctx = input;
control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
( 1 << XHCI_CTX_EP0 ) );
/* Populate slot context */
slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( 1, 0, slot->psiv,
slot->route ) );
slot_ctx->port = slot->port;
/* Populate control endpoint context */
ep_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_EP0 ) );
ep_ctx->type = XHCI_EP_TYPE_CONTROL;
ep_ctx->burst = endpoint->ep->burst;
ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
ep_ctx->dequeue = cpu_to_le64 ( virt_to_phys ( endpoint->ring.trb ) |
XHCI_EP_DCS );
ep_ctx->trb_len = cpu_to_le16 ( XHCI_EP0_TRB_LEN );
}
/**
* Address device
*
* @v xhci xHCI device
* @v slot Device slot
* @ret rc Return status code
*/
static inline int xhci_address_device ( struct xhci_device *xhci,
struct xhci_slot *slot ) {
struct usb_device *usb = slot->usb;
struct xhci_slot_context *slot_ctx;
int rc;
/* Assign device address */
if ( ( rc = xhci_context ( xhci, slot, slot->endpoint[XHCI_CTX_EP0],
XHCI_TRB_ADDRESS_DEVICE,
xhci_address_device_input ) ) != 0 )
return rc;
/* Get assigned address */
slot_ctx = ( slot->context +
xhci_device_context_offset ( xhci, XHCI_CTX_SLOT ) );
usb->address = slot_ctx->address;
DBGC2 ( xhci, "XHCI %p assigned address %d to %s\n",
xhci, usb->address, usb->name );
return 0;
}
/**
* Populate configure endpoint input context
*
* @v xhci xHCI device
* @v slot Device slot
* @v endpoint Endpoint
* @v input Input context
*/
static void xhci_configure_endpoint_input ( struct xhci_device *xhci,
struct xhci_slot *slot __unused,
struct xhci_endpoint *endpoint,
void *input ) {
struct xhci_control_context *control_ctx;
struct xhci_slot_context *slot_ctx;
struct xhci_endpoint_context *ep_ctx;
/* Populate control context */
control_ctx = input;
control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
( 1 << endpoint->ctx ) );
/* Populate slot context */
slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
0, 0, 0 ) );
/* Populate endpoint context */
ep_ctx = ( input + xhci_input_context_offset ( xhci, endpoint->ctx ) );
ep_ctx->interval = endpoint->interval;
ep_ctx->type = endpoint->type;
ep_ctx->burst = endpoint->ep->burst;
ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
ep_ctx->dequeue = cpu_to_le64 ( virt_to_phys ( endpoint->ring.trb ) |
XHCI_EP_DCS );
ep_ctx->trb_len = cpu_to_le16 ( endpoint->ep->mtu ); /* best guess */
}
/**
* Configure endpoint
*
* @v xhci xHCI device
* @v slot Device slot
* @v endpoint Endpoint
* @ret rc Return status code
*/
static inline int xhci_configure_endpoint ( struct xhci_device *xhci,
struct xhci_slot *slot,
struct xhci_endpoint *endpoint ) {
int rc;
/* Configure endpoint */
if ( ( rc = xhci_context ( xhci, slot, endpoint,
XHCI_TRB_CONFIGURE_ENDPOINT,
xhci_configure_endpoint_input ) ) != 0 )
return rc;
DBGC2 ( xhci, "XHCI %p slot %d ctx %d configured\n",
xhci, slot->id, endpoint->ctx );
return 0;
}
/**
* Populate deconfigure endpoint input context
*
* @v xhci xHCI device
* @v slot Device slot
* @v endpoint Endpoint
* @v input Input context
*/
static void
xhci_deconfigure_endpoint_input ( struct xhci_device *xhci __unused,
struct xhci_slot *slot __unused,
struct xhci_endpoint *endpoint,
void *input ) {
struct xhci_control_context *control_ctx;
struct xhci_slot_context *slot_ctx;
/* Populate control context */
control_ctx = input;
control_ctx->add = cpu_to_le32 ( 1 << XHCI_CTX_SLOT );
control_ctx->drop = cpu_to_le32 ( 1 << endpoint->ctx );
/* Populate slot context */
slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
0, 0, 0 ) );
}
/**
* Deconfigure endpoint
*
* @v xhci xHCI device
* @v slot Device slot
* @v endpoint Endpoint
* @ret rc Return status code
*/
static inline int xhci_deconfigure_endpoint ( struct xhci_device *xhci,
struct xhci_slot *slot,
struct xhci_endpoint *endpoint ) {
int rc;
/* Deconfigure endpoint */
if ( ( rc = xhci_context ( xhci, slot, endpoint,
XHCI_TRB_CONFIGURE_ENDPOINT,
xhci_deconfigure_endpoint_input ) ) != 0 )
return rc;
DBGC2 ( xhci, "XHCI %p slot %d ctx %d deconfigured\n",
xhci, slot->id, endpoint->ctx );
return 0;
}
/**
* Populate evaluate context input context
*
* @v xhci xHCI device
* @v slot Device slot
* @v endpoint Endpoint
* @v input Input context
*/
static void xhci_evaluate_context_input ( struct xhci_device *xhci,
struct xhci_slot *slot __unused,
struct xhci_endpoint *endpoint,
void *input ) {
struct xhci_control_context *control_ctx;
struct xhci_slot_context *slot_ctx;
struct xhci_endpoint_context *ep_ctx;
/* Populate control context */
control_ctx = input;
control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
( 1 << endpoint->ctx ) );
/* Populate slot context */
slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
0, 0, 0 ) );
/* Populate endpoint context */
ep_ctx = ( input + xhci_input_context_offset ( xhci, endpoint->ctx ) );
ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
}
/**
* Evaluate context
*
* @v xhci xHCI device
* @v slot Device slot
* @v endpoint Endpoint
* @ret rc Return status code
*/
static inline int xhci_evaluate_context ( struct xhci_device *xhci,
struct xhci_slot *slot,
struct xhci_endpoint *endpoint ) {
int rc;
/* Configure endpoint */
if ( ( rc = xhci_context ( xhci, slot, endpoint,
XHCI_TRB_EVALUATE_CONTEXT,
xhci_evaluate_context_input ) ) != 0 )
return rc;
DBGC2 ( xhci, "XHCI %p slot %d ctx %d (re-)evaluated\n",
xhci, slot->id, endpoint->ctx );
return 0;
}
/**
* Reset endpoint
*
* @v xhci xHCI device
* @v slot Device slot
* @v endpoint Endpoint
* @ret rc Return status code
*/
static inline int xhci_reset_endpoint ( struct xhci_device *xhci,
struct xhci_slot *slot,
struct xhci_endpoint *endpoint ) {
union xhci_trb trb;
struct xhci_trb_reset_endpoint *reset = &trb.reset;
int rc;
/* Construct command */
memset ( reset, 0, sizeof ( *reset ) );
reset->slot = slot->id;
reset->endpoint = endpoint->ctx;
reset->type = XHCI_TRB_RESET_ENDPOINT;
/* Issue command and wait for completion */
if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
DBGC ( xhci, "XHCI %p slot %d ctx %d could not reset endpoint "
"in state %d: %s\n", xhci, slot->id, endpoint->ctx,
endpoint->context->state, strerror ( rc ) );
return rc;
}
return 0;
}
/**
* Stop endpoint
*
* @v xhci xHCI device
* @v slot Device slot
* @v endpoint Endpoint
* @ret rc Return status code
*/
static inline int xhci_stop_endpoint ( struct xhci_device *xhci,
struct xhci_slot *slot,
struct xhci_endpoint *endpoint ) {
union xhci_trb trb;
struct xhci_trb_stop_endpoint *stop = &trb.stop;
int rc;
/* Construct command */
memset ( stop, 0, sizeof ( *stop ) );
stop->slot = slot->id;
stop->endpoint = endpoint->ctx;
stop->type = XHCI_TRB_STOP_ENDPOINT;
/* Issue command and wait for completion */
if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
DBGC ( xhci, "XHCI %p slot %d ctx %d could not stop endpoint "
"in state %d: %s\n", xhci, slot->id, endpoint->ctx,
endpoint->context->state, strerror ( rc ) );
return rc;
}
return 0;
}
/**
* Set transfer ring dequeue pointer
*
* @v xhci xHCI device
* @v slot Device slot
* @v endpoint Endpoint
* @ret rc Return status code
*/
static inline int
xhci_set_tr_dequeue_pointer ( struct xhci_device *xhci,
struct xhci_slot *slot,
struct xhci_endpoint *endpoint ) {
union xhci_trb trb;
struct xhci_trb_set_tr_dequeue_pointer *dequeue = &trb.dequeue;
struct xhci_trb_ring *ring = &endpoint->ring;
unsigned int cons;
unsigned int mask;
unsigned int index;
unsigned int dcs;
int rc;
/* Construct command */
memset ( dequeue, 0, sizeof ( *dequeue ) );
cons = ring->cons;
mask = ring->mask;
dcs = ( ( ~( cons >> ring->shift ) ) & XHCI_EP_DCS );
index = ( cons & mask );
dequeue->dequeue =
cpu_to_le64 ( virt_to_phys ( &ring->trb[index] ) | dcs );
dequeue->slot = slot->id;
dequeue->endpoint = endpoint->ctx;
dequeue->type = XHCI_TRB_SET_TR_DEQUEUE_POINTER;
/* Issue command and wait for completion */
if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
DBGC ( xhci, "XHCI %p slot %d ctx %d could not set TR dequeue "
"pointer in state %d: %s\n", xhci, slot->id,
endpoint->ctx, endpoint->context->state, strerror ( rc));
return rc;
}
return 0;
}
/******************************************************************************
*
* Endpoint operations
*
******************************************************************************
*/
/**
* Open endpoint
*
* @v ep USB endpoint
* @ret rc Return status code
*/
static int xhci_endpoint_open ( struct usb_endpoint *ep ) {
struct usb_device *usb = ep->usb;
struct xhci_slot *slot = usb_get_hostdata ( usb );
struct xhci_device *xhci = slot->xhci;
struct xhci_endpoint *endpoint;
unsigned int ctx;
unsigned int type;
unsigned int interval;
int rc;
/* Calculate context index */
ctx = XHCI_CTX ( ep->address );
assert ( slot->endpoint[ctx] == NULL );
/* Calculate endpoint type */
type = XHCI_EP_TYPE ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
if ( type == XHCI_EP_TYPE ( USB_ENDPOINT_ATTR_CONTROL ) )
type = XHCI_EP_TYPE_CONTROL;
if ( ep->address & USB_DIR_IN )
type |= XHCI_EP_TYPE_IN;
/* Calculate interval */
if ( type & XHCI_EP_TYPE_PERIODIC ) {
interval = ( fls ( ep->interval ) - 1 );
} else {
interval = ep->interval;
}
/* Allocate and initialise structure */
endpoint = zalloc ( sizeof ( *endpoint ) );
if ( ! endpoint ) {
rc = -ENOMEM;
goto err_alloc;
}
usb_endpoint_set_hostdata ( ep, endpoint );
slot->endpoint[ctx] = endpoint;
endpoint->xhci = xhci;
endpoint->slot = slot;
endpoint->ep = ep;
endpoint->ctx = ctx;
endpoint->type = type;
endpoint->interval = interval;
endpoint->context = ( ( ( void * ) slot->context ) +
xhci_device_context_offset ( xhci, ctx ) );
/* Allocate transfer ring */
if ( ( rc = xhci_ring_alloc ( xhci, &endpoint->ring,
XHCI_TRANSFER_TRBS_LOG2,
slot->id, ctx, 0 ) ) != 0 )
goto err_ring_alloc;
/* Configure endpoint, if applicable */
if ( ( ctx != XHCI_CTX_EP0 ) &&
( ( rc = xhci_configure_endpoint ( xhci, slot, endpoint ) ) != 0 ))
goto err_configure_endpoint;
DBGC2 ( xhci, "XHCI %p slot %d ctx %d ring [%08lx,%08lx)\n",
xhci, slot->id, ctx, virt_to_phys ( endpoint->ring.trb ),
( virt_to_phys ( endpoint->ring.trb ) + endpoint->ring.len ) );
return 0;
xhci_deconfigure_endpoint ( xhci, slot, endpoint );
err_configure_endpoint:
xhci_ring_free ( &endpoint->ring );
err_ring_alloc:
slot->endpoint[ctx] = NULL;
free ( endpoint );
err_alloc:
return rc;
}
/**
* Close endpoint
*
* @v ep USB endpoint
*/
static void xhci_endpoint_close ( struct usb_endpoint *ep ) {
struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
struct xhci_slot *slot = endpoint->slot;
struct xhci_device *xhci = slot->xhci;
struct io_buffer *iobuf;
unsigned int ctx = endpoint->ctx;
/* Deconfigure endpoint, if applicable */
if ( ctx != XHCI_CTX_EP0 )
xhci_deconfigure_endpoint ( xhci, slot, endpoint );
/* Cancel any incomplete transfers */
while ( xhci_ring_fill ( &endpoint->ring ) ) {
iobuf = xhci_dequeue_multi ( &endpoint->ring );
usb_complete_err ( ep, iobuf, -ECANCELED );
}
/* Free endpoint */
xhci_ring_free ( &endpoint->ring );
slot->endpoint[ctx] = NULL;
free ( endpoint );
}
/**
* Reset endpoint
*
* @v ep USB endpoint
* @ret rc Return status code
*/
static int xhci_endpoint_reset ( struct usb_endpoint *ep ) {
struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
struct xhci_slot *slot = endpoint->slot;
struct xhci_device *xhci = slot->xhci;
int rc;
/* Reset endpoint context */
if ( ( rc = xhci_reset_endpoint ( xhci, slot, endpoint ) ) != 0 )
return rc;
/* Set transfer ring dequeue pointer */
if ( ( rc = xhci_set_tr_dequeue_pointer ( xhci, slot, endpoint ) ) != 0)
return rc;
DBGC ( xhci, "XHCI %p slot %d ctx %d reset\n",
xhci, slot->id, endpoint->ctx );
return 0;
}
/**
* Update MTU
*
* @v ep USB endpoint
* @ret rc Return status code
*/
static int xhci_endpoint_mtu ( struct usb_endpoint *ep ) {
struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
struct xhci_slot *slot = endpoint->slot;
struct xhci_device *xhci = slot->xhci;
int rc;
/* Evalulate context */
if ( ( rc = xhci_evaluate_context ( xhci, slot, endpoint ) ) != 0 )
return rc;
return 0;
}
/**
* Enqueue message transfer
*
* @v ep USB endpoint
* @v packet Setup packet
* @v iobuf I/O buffer
* @ret rc Return status code
*/
static int xhci_endpoint_message ( struct usb_endpoint *ep,
struct usb_setup_packet *packet,
struct io_buffer *iobuf ) {
struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
unsigned int input = ( le16_to_cpu ( packet->request ) & USB_DIR_IN );
size_t len = iob_len ( iobuf );
union xhci_trb trbs[ 1 /* setup */ + 1 /* possible data */ +
1 /* status */ ];
union xhci_trb *trb = trbs;
struct xhci_trb_setup *setup;
struct xhci_trb_data *data;
struct xhci_trb_status *status;
int rc;
/* Profile message transfers */
profile_start ( &xhci_message_profiler );
/* Construct setup stage TRB */
memset ( trbs, 0, sizeof ( trbs ) );
setup = &(trb++)->setup;
memcpy ( &setup->packet, packet, sizeof ( setup->packet ) );
setup->len = cpu_to_le32 ( sizeof ( *packet ) );
setup->flags = XHCI_TRB_IDT;
setup->type = XHCI_TRB_SETUP;
if ( len )
setup->direction = ( input ? XHCI_SETUP_IN : XHCI_SETUP_OUT );
/* Construct data stage TRB, if applicable */
if ( len ) {
data = &(trb++)->data;
data->data = cpu_to_le64 ( virt_to_phys ( iobuf->data ) );
data->len = cpu_to_le32 ( len );
data->type = XHCI_TRB_DATA;
data->direction = ( input ? XHCI_DATA_IN : XHCI_DATA_OUT );
}
/* Construct status stage TRB */
status = &(trb++)->status;
status->flags = XHCI_TRB_IOC;
status->type = XHCI_TRB_STATUS;
status->direction =
( ( len && input ) ? XHCI_STATUS_OUT : XHCI_STATUS_IN );
/* Enqueue TRBs */
if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs,
( trb - trbs ) ) ) != 0 )
return rc;
/* Ring the doorbell */
xhci_doorbell ( &endpoint->ring );
profile_stop ( &xhci_message_profiler );
return 0;
}
/**
* Enqueue stream transfer
*
* @v ep USB endpoint
* @v iobuf I/O buffer
* @v terminate Terminate using a short packet
* @ret rc Return status code
*/
static int xhci_endpoint_stream ( struct usb_endpoint *ep,
struct io_buffer *iobuf, int terminate ) {
struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
union xhci_trb trbs[ 1 /* Normal */ + 1 /* Possible zero-length */ ];
union xhci_trb *trb = trbs;
struct xhci_trb_normal *normal;
size_t len = iob_len ( iobuf );
int rc;
/* Profile stream transfers */
profile_start ( &xhci_stream_profiler );
/* Construct normal TRBs */
memset ( &trbs, 0, sizeof ( trbs ) );
normal = &(trb++)->normal;
normal->data = cpu_to_le64 ( virt_to_phys ( iobuf->data ) );
normal->len = cpu_to_le32 ( len );
normal->type = XHCI_TRB_NORMAL;
if ( terminate && ( ( len & ( ep->mtu - 1 ) ) == 0 ) ) {
normal->flags = XHCI_TRB_CH;
normal = &(trb++)->normal;
normal->type = XHCI_TRB_NORMAL;
}
normal->flags = XHCI_TRB_IOC;
/* Enqueue TRBs */
if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs,
( trb - trbs ) ) ) != 0 )
return rc;
/* Ring the doorbell */
xhci_doorbell ( &endpoint->ring );
profile_stop ( &xhci_stream_profiler );
return 0;
}
/******************************************************************************
*
* Device operations
*
******************************************************************************
*/
/**
* Open device
*
* @v usb USB device
* @ret rc Return status code
*/
static int xhci_device_open ( struct usb_device *usb ) {
struct xhci_device *xhci = usb_bus_get_hostdata ( usb->port->hub->bus );
struct xhci_slot *slot;
size_t len;
int type;
int id;
int rc;
/* Determine applicable slot type */
type = xhci_port_slot_type ( xhci, usb->port->address );
if ( type < 0 ) {
rc = type;
DBGC ( xhci, "XHCI %p port %d has no slot type\n",
xhci, usb->port->address );
goto err_type;
}
/* Allocate a device slot number */
id = xhci_enable_slot ( xhci, type );
if ( id < 0 ) {
rc = id;
goto err_enable_slot;
}
assert ( xhci->slot[id] == NULL );
/* Allocate and initialise structure */
slot = zalloc ( sizeof ( *slot ) );
if ( ! slot ) {
rc = -ENOMEM;
goto err_alloc;
}
usb_set_hostdata ( usb, slot );
xhci->slot[id] = slot;
slot->xhci = xhci;
slot->usb = usb;
slot->id = id;
/* Allocate a device context */
len = xhci_device_context_offset ( xhci, XHCI_CTX_END );
slot->context = malloc_dma ( len, xhci_align ( len ) );
if ( ! slot->context ) {
rc = -ENOMEM;
goto err_alloc_context;
}
memset ( slot->context, 0, len );
/* Set device context base address */
assert ( xhci->dcbaa[id] == 0 );
xhci->dcbaa[id] = cpu_to_le64 ( virt_to_phys ( slot->context ) );
DBGC2 ( xhci, "XHCI %p slot %d device context [%08lx,%08lx) for %s\n",
xhci, slot->id, virt_to_phys ( slot->context ),
( virt_to_phys ( slot->context ) + len ), usb->name );
return 0;
xhci->dcbaa[id] = 0;
free_dma ( slot->context, len );
err_alloc_context:
xhci->slot[id] = NULL;
free ( slot );
err_alloc:
xhci_disable_slot ( xhci, id );
err_enable_slot:
err_type:
return rc;
}
/**
* Close device
*
* @v usb USB device
*/
static void xhci_device_close ( struct usb_device *usb ) {
struct xhci_slot *slot = usb_get_hostdata ( usb );
struct xhci_device *xhci = slot->xhci;
size_t len = xhci_device_context_offset ( xhci, XHCI_CTX_END );
unsigned int id = slot->id;
/* Disable slot */
xhci_disable_slot ( xhci, id );
/* Free slot */
free_dma ( slot->context, len );
xhci->dcbaa[id] = 0;
xhci->slot[id] = NULL;
free ( slot );
}
/**
* Assign device address
*
* @v usb USB device
* @ret rc Return status code
*/
static int xhci_device_address ( struct usb_device *usb ) {
struct xhci_slot *slot = usb_get_hostdata ( usb );
struct xhci_device *xhci = slot->xhci;
struct usb_port *port = usb->port;
struct usb_port *root_port;
int psiv;
int rc;
/* Calculate route string */
slot->route = usb_route_string ( usb );
/* Calculate root hub port number */
root_port = usb_root_hub_port ( usb );
slot->port = root_port->address;
/* Calculate protocol speed ID */
psiv = xhci_port_psiv ( xhci, slot->port, port->speed );
if ( psiv < 0 ) {
rc = psiv;
return rc;
}
slot->psiv = psiv;
/* Address device */
if ( ( rc = xhci_address_device ( xhci, slot ) ) != 0 )
return rc;
return 0;
}
/******************************************************************************
*
* Bus operations
*
******************************************************************************
*/
/**
* Open USB bus
*
* @v bus USB bus
* @ret rc Return status code
*/
static int xhci_bus_open ( struct usb_bus *bus ) {
struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
int rc;
/* Allocate device slot array */
xhci->slot = zalloc ( xhci->slots * sizeof ( xhci->slot[0] ) );
if ( ! xhci->slot ) {
rc = -ENOMEM;
goto err_slot_alloc;
}
/* Allocate device context base address array */
if ( ( rc = xhci_dcbaa_alloc ( xhci ) ) != 0 )
goto err_dcbaa_alloc;
/* Allocate scratchpad buffers */
if ( ( rc = xhci_scratchpad_alloc ( xhci ) ) != 0 )
goto err_scratchpad_alloc;
/* Allocate command ring */
if ( ( rc = xhci_command_alloc ( xhci ) ) != 0 )
goto err_command_alloc;
/* Allocate event ring */
if ( ( rc = xhci_event_alloc ( xhci ) ) != 0 )
goto err_event_alloc;
/* Start controller */
xhci_run ( xhci );
return 0;
xhci_stop ( xhci );
xhci_event_free ( xhci );
err_event_alloc:
xhci_command_free ( xhci );
err_command_alloc:
xhci_scratchpad_free ( xhci );
err_scratchpad_alloc:
xhci_dcbaa_free ( xhci );
err_dcbaa_alloc:
free ( xhci->slot );
err_slot_alloc:
return rc;
}
/**
* Close USB bus
*
* @v bus USB bus
*/
static void xhci_bus_close ( struct usb_bus *bus ) {
struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
unsigned int i;
/* Sanity checks */
assert ( xhci->slot != NULL );
for ( i = 0 ; i < xhci->slots ; i++ )
assert ( xhci->slot[i] == NULL );
xhci_stop ( xhci );
xhci_event_free ( xhci );
xhci_command_free ( xhci );
xhci_scratchpad_free ( xhci );
xhci_dcbaa_free ( xhci );
free ( xhci->slot );
}
/**
* Poll USB bus
*
* @v bus USB bus
*/
static void xhci_bus_poll ( struct usb_bus *bus ) {
struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
/* Poll event ring */
xhci_event_poll ( xhci );
}
/******************************************************************************
*
* Root hub operations
*
******************************************************************************
*/
/**
* Open root hub
*
* @v hub USB hub
* @ret rc Return status code
*/
static int xhci_hub_open ( struct usb_hub *hub ) {
struct usb_bus *bus = hub->bus;
struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
struct usb_port *port;
uint32_t portsc;
unsigned int i;
/* Enable power to all ports */
for ( i = 1 ; i <= xhci->ports ; i++ ) {
portsc = readl ( xhci->op + XHCI_OP_PORTSC ( i ) );
portsc &= XHCI_PORTSC_PRESERVE;
portsc |= XHCI_PORTSC_PP;
writel ( portsc, xhci->op + XHCI_OP_PORTSC ( i ) );
}
/* xHCI spec requires us to potentially wait 20ms after
* enabling power to a port.
*/
mdelay ( XHCI_PORT_POWER_DELAY_MS );
/* USB3 ports may power up as Disabled */
for ( i = 1 ; i <= xhci->ports ; i++ ) {
portsc = readl ( xhci->op + XHCI_OP_PORTSC ( i ) );
port = usb_port ( hub, i );
if ( ( port->protocol >= USB_PROTO_3_0 ) &&
( ( portsc & XHCI_PORTSC_PLS_MASK ) ==
XHCI_PORTSC_PLS_DISABLED ) ) {
/* Force link state to RxDetect */
portsc &= XHCI_PORTSC_PRESERVE;
portsc |= ( XHCI_PORTSC_PLS_RXDETECT | XHCI_PORTSC_LWS);
writel ( portsc, xhci->op + XHCI_OP_PORTSC ( i ) );
}
}
/* Some xHCI cards seem to require an additional delay after
* setting the link state to RxDetect.
*/
mdelay ( XHCI_LINK_STATE_DELAY_MS );
/* Record hub driver private data */
usb_hub_set_drvdata ( hub, xhci );
return 0;
}
/**
* Close root hub
*
* @v hub USB hub
*/
static void xhci_hub_close ( struct usb_hub *hub ) {
/* Clear hub driver private data */
usb_hub_set_drvdata ( hub, NULL );
}
/**
* Enable port
*
* @v hub USB hub
* @v port USB port
* @ret rc Return status code
*/
static int xhci_hub_enable ( struct usb_hub *hub, struct usb_port *port ) {
struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
uint32_t portsc;
unsigned int i;
/* Reset port if applicable */
if ( port->protocol < USB_PROTO_3_0 ) {
portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
portsc &= XHCI_PORTSC_PRESERVE;
portsc |= XHCI_PORTSC_PR;
writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
}
/* Wait for port to become enabled */
for ( i = 0 ; i < XHCI_PORT_RESET_MAX_WAIT_MS ; i++ ) {
/* Check port status */
portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
if ( portsc & XHCI_PORTSC_PED )
return 0;
/* Delay */
mdelay ( 1 );
}
DBGC ( xhci, "XHCI %p timed out waiting for port %d to enable\n",
xhci, port->address );
return -ETIMEDOUT;
}
/**
* Disable port
*
* @v hub USB hub
* @v port USB port
* @ret rc Return status code
*/
static int xhci_hub_disable ( struct usb_hub *hub, struct usb_port *port ) {
struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
uint32_t portsc;
/* Disable port */
portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
portsc &= XHCI_PORTSC_PRESERVE;
portsc |= XHCI_PORTSC_PED;
writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
return 0;
}
/**
* Update root hub port speed
*
* @v hub USB hub
* @v port USB port
* @ret rc Return status code
*/
static int xhci_hub_speed ( struct usb_hub *hub, struct usb_port *port ) {
struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
uint32_t portsc;
unsigned int psiv;
int ccs;
int ped;
int speed;
int rc;
/* Read port status */
portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
DBGC2 ( xhci, "XHCI %p port %d status is %08x\n",
xhci, port->address, portsc );
/* Check whether or not port is connected */
ccs = ( portsc & XHCI_PORTSC_CCS );
if ( ! ccs ) {
port->speed = USB_SPEED_NONE;
return 0;
}
/* For USB2 ports, the PSIV field is not valid until the port
* completes reset and becomes enabled.
*/
ped = ( portsc & XHCI_PORTSC_PED );
if ( ( port->protocol < USB_PROTO_3_0 ) && ! ped ) {
port->speed = USB_SPEED_FULL;
return 0;
}
/* Get port speed and map to generic USB speed */
psiv = XHCI_PORTSC_PSIV ( portsc );
speed = xhci_port_speed ( xhci, port->address, psiv );
if ( speed < 0 ) {
rc = speed;
return rc;
}
port->speed = speed;
return 0;
}
/******************************************************************************
*
* PCI interface
*
******************************************************************************
*/
/** USB host controller operations */
static struct usb_host_operations xhci_operations = {
.endpoint = {
.open = xhci_endpoint_open,
.close = xhci_endpoint_close,
.reset = xhci_endpoint_reset,
.mtu = xhci_endpoint_mtu,
.message = xhci_endpoint_message,
.stream = xhci_endpoint_stream,
},
.device = {
.open = xhci_device_open,
.close = xhci_device_close,
.address = xhci_device_address,
},
.bus = {
.open = xhci_bus_open,
.close = xhci_bus_close,
.poll = xhci_bus_poll,
},
.hub = {
.open = xhci_hub_open,
.close = xhci_hub_close,
.enable = xhci_hub_enable,
.disable = xhci_hub_disable,
.speed = xhci_hub_speed,
},
};
/**
* Probe PCI device
*
* @v pci PCI device
* @ret rc Return status code
*/
static int xhci_probe ( struct pci_device *pci ) {
struct xhci_device *xhci;
struct usb_port *port;
unsigned long bar_start;
size_t bar_size;
unsigned int i;
int rc;
/* Allocate and initialise structure */
xhci = zalloc ( sizeof ( *xhci ) );
if ( ! xhci ) {
rc = -ENOMEM;
goto err_alloc;
}
/* Fix up PCI device */
adjust_pci_device ( pci );
/* Map registers */
bar_start = pci_bar_start ( pci, XHCI_BAR );
bar_size = pci_bar_size ( pci, XHCI_BAR );
xhci->regs = ioremap ( bar_start, bar_size );
if ( ! xhci->regs ) {
rc = -ENODEV;
goto err_ioremap;
}
/* Initialise xHCI device */
xhci_init ( xhci, xhci->regs );
/* Initialise USB legacy support and claim ownership */
xhci_legacy_init ( xhci );
if ( ( rc = xhci_legacy_claim ( xhci ) ) != 0 )
goto err_legacy_claim;
/* Reset device */
if ( ( rc = xhci_reset ( xhci ) ) != 0 )
goto err_reset;
/* Allocate USB bus */
xhci->bus = alloc_usb_bus ( &pci->dev, xhci->ports,
&xhci_operations );
if ( ! xhci->bus ) {
rc = -ENOMEM;
goto err_alloc_bus;
}
usb_bus_set_hostdata ( xhci->bus, xhci );
usb_hub_set_drvdata ( xhci->bus->hub, xhci );
/* Set port protocols */
for ( i = 1 ; i <= xhci->ports ; i++ ) {
port = usb_port ( xhci->bus->hub, i );
port->protocol = xhci_port_protocol ( xhci, i );
}
/* Register USB bus */
if ( ( rc = register_usb_bus ( xhci->bus ) ) != 0 )
goto err_register;
pci_set_drvdata ( pci, xhci );
return 0;
unregister_usb_bus ( xhci->bus );
err_register:
free_usb_bus ( xhci->bus );
err_alloc_bus:
xhci_reset ( xhci );
err_reset:
xhci_legacy_release ( xhci );
err_legacy_claim:
iounmap ( xhci->regs );
err_ioremap:
free ( xhci );
err_alloc:
return rc;
}
/**
* Remove PCI device
*
* @v pci PCI device
*/
static void xhci_remove ( struct pci_device *pci ) {
struct xhci_device *xhci = pci_get_drvdata ( pci );
struct usb_bus *bus = xhci->bus;
unregister_usb_bus ( bus );
free_usb_bus ( bus );
xhci_reset ( xhci );
xhci_legacy_release ( xhci );
iounmap ( xhci->regs );
free ( xhci );
}
/** XHCI PCI device IDs */
static struct pci_device_id xhci_ids[] = {
PCI_ROM ( 0xffff, 0xffff, "xhci", "xHCI", 0 ),
};
/** XHCI PCI driver */
struct pci_driver xhci_driver __pci_driver = {
.ids = xhci_ids,
.id_count = ( sizeof ( xhci_ids ) / sizeof ( xhci_ids[0] ) ),
.class = PCI_CLASS ( PCI_CLASS_SERIAL, PCI_CLASS_SERIAL_USB,
PCI_CLASS_SERIAL_USB_XHCI ),
.probe = xhci_probe,
.remove = xhci_remove,
};
