Merge branch 'master' into openslx

7 files changed, 1274 insertions, 129 deletions

diff --git a/src/drivers/usb/ehci.c b/src/drivers/usb/ehci.c
index 29c32b356..77022a47d 100644
--- a/src/drivers/usb/ehci.c
+++ b/src/drivers/usb/ehci.c
@@ -565,8 +565,8 @@ static int ehci_ring_alloc ( struct ehci_device *ehci,
 	}
 
 	/* Allocate queue head */
-	ring->head = malloc_dma ( sizeof ( *ring->head ),
-				  ehci_align ( sizeof ( *ring->head ) ) );
+	ring->head = malloc_phys ( sizeof ( *ring->head ),
+				   ehci_align ( sizeof ( *ring->head ) ) );
 	if ( ! ring->head ) {
 		rc = -ENOMEM;
 		goto err_alloc_queue;
@@ -579,7 +579,7 @@ static int ehci_ring_alloc ( struct ehci_device *ehci,
 
 	/* Allocate transfer descriptors */
 	len = ( EHCI_RING_COUNT * sizeof ( ring->desc[0] ) );
-	ring->desc = malloc_dma ( len, sizeof ( ring->desc[0] ) );
+	ring->desc = malloc_phys ( len, sizeof ( ring->desc[0] ) );
 	if ( ! ring->desc ) {
 		rc = -ENOMEM;
 		goto err_alloc_desc;
@@ -607,10 +607,10 @@ static int ehci_ring_alloc ( struct ehci_device *ehci,
 	return 0;
 
  err_unreachable_desc:
-	free_dma ( ring->desc, len );
+	free_phys ( ring->desc, len );
  err_alloc_desc:
  err_unreachable_queue:
-	free_dma ( ring->head, sizeof ( *ring->head ) );
+	free_phys ( ring->head, sizeof ( *ring->head ) );
  err_alloc_queue:
 	free ( ring->iobuf );
  err_alloc_iobuf:
@@ -631,10 +631,11 @@ static void ehci_ring_free ( struct ehci_ring *ring ) {
 		assert ( ring->iobuf[i] == NULL );
 
 	/* Free transfer descriptors */
-	free_dma ( ring->desc, ( EHCI_RING_COUNT * sizeof ( ring->desc[0] ) ) );
+	free_phys ( ring->desc, ( EHCI_RING_COUNT *
+				  sizeof ( ring->desc[0] ) ) );
 
 	/* Free queue head */
-	free_dma ( ring->head, sizeof ( *ring->head ) );
+	free_phys ( ring->head, sizeof ( *ring->head ) );
 
 	/* Free I/O buffers */
 	free ( ring->iobuf );
@@ -1787,8 +1788,8 @@ static int ehci_bus_open ( struct usb_bus *bus ) {
 	assert ( list_empty ( &ehci->periodic ) );
 
 	/* Allocate and initialise asynchronous queue head */
-	ehci->head = malloc_dma ( sizeof ( *ehci->head ),
-				  ehci_align ( sizeof ( *ehci->head ) ) );
+	ehci->head = malloc_phys ( sizeof ( *ehci->head ),
+				   ehci_align ( sizeof ( *ehci->head ) ) );
 	if ( ! ehci->head ) {
 		rc = -ENOMEM;
 		goto err_alloc_head;
@@ -1816,7 +1817,7 @@ static int ehci_bus_open ( struct usb_bus *bus ) {
 	/* Allocate periodic frame list */
 	frames = EHCI_PERIODIC_FRAMES ( ehci->flsize );
 	len = ( frames * sizeof ( ehci->frame[0] ) );
-	ehci->frame = malloc_dma ( len, EHCI_PAGE_ALIGN );
+	ehci->frame = malloc_phys ( len, EHCI_PAGE_ALIGN );
 	if ( ! ehci->frame ) {
 		rc = -ENOMEM;
 		goto err_alloc_frame;
@@ -1836,10 +1837,10 @@ static int ehci_bus_open ( struct usb_bus *bus ) {
 
 	ehci_stop ( ehci );
  err_unreachable_frame:
-	free_dma ( ehci->frame, len );
+	free_phys ( ehci->frame, len );
  err_alloc_frame:
  err_ctrldssegment:
-	free_dma ( ehci->head, sizeof ( *ehci->head ) );
+	free_phys ( ehci->head, sizeof ( *ehci->head ) );
  err_alloc_head:
 	return rc;
 }
@@ -1861,10 +1862,10 @@ static void ehci_bus_close ( struct usb_bus *bus ) {
 	ehci_stop ( ehci );
 
 	/* Free periodic frame list */
-	free_dma ( ehci->frame, ( frames * sizeof ( ehci->frame[0] ) ) );
+	free_phys ( ehci->frame, ( frames * sizeof ( ehci->frame[0] ) ) );
 
 	/* Free asynchronous schedule */
-	free_dma ( ehci->head, sizeof ( *ehci->head ) );
+	free_phys ( ehci->head, sizeof ( *ehci->head ) );
 }
 
 /**
@@ -1989,7 +1990,7 @@ static int ehci_probe ( struct pci_device *pci ) {
 	/* Map registers */
 	bar_start = pci_bar_start ( pci, EHCI_BAR );
 	bar_size = pci_bar_size ( pci, EHCI_BAR );
-	ehci->regs = ioremap ( bar_start, bar_size );
+	ehci->regs = pci_ioremap ( pci, bar_start, bar_size );
 	if ( ! ehci->regs ) {
 		rc = -ENODEV;
 		goto err_ioremap;
diff --git a/src/drivers/usb/uhci.c b/src/drivers/usb/uhci.c
index ce2962d36..47474bdc7 100644
--- a/src/drivers/usb/uhci.c
+++ b/src/drivers/usb/uhci.c
@@ -179,7 +179,7 @@ static int uhci_ring_alloc ( struct uhci_ring *ring ) {
 	memset ( ring, 0, sizeof ( *ring ) );
 
 	/* Allocate queue head */
-	ring->head = malloc_dma ( sizeof ( *ring->head ), UHCI_ALIGN );
+	ring->head = malloc_phys ( sizeof ( *ring->head ), UHCI_ALIGN );
 	if ( ! ring->head ) {
 		rc = -ENOMEM;
 		goto err_alloc;
@@ -194,7 +194,7 @@ static int uhci_ring_alloc ( struct uhci_ring *ring ) {
 	return 0;
 
  err_unreachable:
-	free_dma ( ring->head, sizeof ( *ring->head ) );
+	free_phys ( ring->head, sizeof ( *ring->head ) );
  err_alloc:
 	return rc;
 }
@@ -213,7 +213,7 @@ static void uhci_ring_free ( struct uhci_ring *ring ) {
 		assert ( ring->xfer[i] == NULL );
 
 	/* Free queue head */
-	free_dma ( ring->head, sizeof ( *ring->head ) );
+	free_phys ( ring->head, sizeof ( *ring->head ) );
 }
 
 /**
@@ -263,7 +263,7 @@ static int uhci_enqueue ( struct uhci_ring *ring, struct io_buffer *iobuf,
 
 	/* Allocate transfer descriptors */
 	len = ( count * sizeof ( xfer->desc[0] ) );
-	xfer->desc = malloc_dma ( len, UHCI_ALIGN );
+	xfer->desc = malloc_phys ( len, UHCI_ALIGN );
 	if ( ! xfer->desc ) {
 		rc = -ENOMEM;
 		goto err_alloc_desc;
@@ -299,7 +299,7 @@ static int uhci_enqueue ( struct uhci_ring *ring, struct io_buffer *iobuf,
 	return 0;
 
  err_unreachable_desc:
-	free_dma ( xfer->desc, len );
+	free_phys ( xfer->desc, len );
  err_alloc_desc:
 	free ( xfer );
  err_alloc_xfer:
@@ -377,7 +377,7 @@ static struct io_buffer * uhci_dequeue ( struct uhci_ring *ring ) {
 
 	/* Free transfer descriptors */
 	len = ( xfer->prod * sizeof ( xfer->desc[0] ) );
-	free_dma ( xfer->desc, len );
+	free_phys ( xfer->desc, len );
 
 	/* Free transfer */
 	free ( xfer );
@@ -1312,7 +1312,7 @@ static int uhci_bus_open ( struct usb_bus *bus ) {
 	assert ( list_empty ( &uhci->periodic ) );
 
 	/* Allocate and initialise asynchronous queue head */
-	uhci->head = malloc_dma ( sizeof ( *uhci->head ), UHCI_ALIGN );
+	uhci->head = malloc_phys ( sizeof ( *uhci->head ), UHCI_ALIGN );
 	if ( ! uhci->head ) {
 		rc = -ENOMEM;
 		goto err_alloc_head;
@@ -1324,8 +1324,8 @@ static int uhci_bus_open ( struct usb_bus *bus ) {
 	uhci_async_schedule ( uhci );
 
 	/* Allocate periodic frame list */
-	uhci->frame = malloc_dma ( sizeof ( *uhci->frame ),
-				   sizeof ( *uhci->frame ) );
+	uhci->frame = malloc_phys ( sizeof ( *uhci->frame ),
+				    sizeof ( *uhci->frame ) );
 	if ( ! uhci->frame ) {
 		rc = -ENOMEM;
 		goto err_alloc_frame;
@@ -1343,10 +1343,10 @@ static int uhci_bus_open ( struct usb_bus *bus ) {
 
 	uhci_stop ( uhci );
  err_unreachable_frame:
-	free_dma ( uhci->frame, sizeof ( *uhci->frame ) );
+	free_phys ( uhci->frame, sizeof ( *uhci->frame ) );
  err_alloc_frame:
  err_unreachable_head:
-	free_dma ( uhci->head, sizeof ( *uhci->head ) );
+	free_phys ( uhci->head, sizeof ( *uhci->head ) );
  err_alloc_head:
 	return rc;
 }
@@ -1367,10 +1367,10 @@ static void uhci_bus_close ( struct usb_bus *bus ) {
 	uhci_stop ( uhci );
 
 	/* Free periodic frame list */
-	free_dma ( uhci->frame, sizeof ( *uhci->frame ) );
+	free_phys ( uhci->frame, sizeof ( *uhci->frame ) );
 
 	/* Free asynchronous schedule */
-	free_dma ( uhci->head, sizeof ( *uhci->head ) );
+	free_phys ( uhci->head, sizeof ( *uhci->head ) );
 }
 
 /**
diff --git a/src/drivers/usb/usbblk.c b/src/drivers/usb/usbblk.c
new file mode 100644
index 000000000..5a086d3f8
--- /dev/null
+++ b/src/drivers/usb/usbblk.c
@@ -0,0 +1,912 @@
+/*
+ * Copyright (C) 2020 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <ipxe/usb.h>
+#include <ipxe/scsi.h>
+#include <ipxe/xfer.h>
+#include <ipxe/uri.h>
+#include <ipxe/open.h>
+#include <ipxe/efi/efi_path.h>
+#include "usbblk.h"
+
+/** @file
+ *
+ * USB mass storage driver
+ *
+ */
+
+static void usbblk_stop ( struct usbblk_device *usbblk, int rc );
+
+/** List of USB block devices */
+static LIST_HEAD ( usbblk_devices );
+
+/******************************************************************************
+ *
+ * Endpoint management
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open endpoints
+ *
+ * @v usbblk		USB block device
+ * @ret rc		Return status code
+ */
+static int usbblk_open ( struct usbblk_device *usbblk ) {
+	struct usb_device *usb = usbblk->func->usb;
+	unsigned int interface = usbblk->func->interface[0];
+	int rc;
+
+	/* Sanity checks */
+	assert ( ! usbblk->in.open );
+	assert ( ! usbblk->out.open );
+
+	/* Issue reset */
+	if ( ( rc = usb_control ( usb, USBBLK_RESET, 0, interface,
+				  NULL, 0 ) ) != 0 ) {
+		DBGC ( usbblk, "USBBLK %s could not issue reset: %s\n",
+		       usbblk->func->name, strerror ( rc ) );
+		goto err_reset;
+	}
+
+	/* Open bulk OUT endpoint */
+	if ( ( rc = usb_endpoint_open ( &usbblk->out ) ) != 0 ) {
+		DBGC ( usbblk, "USBBLK %s could not open bulk OUT: %s\n",
+		       usbblk->func->name, strerror ( rc ) );
+		goto err_open_out;
+	}
+
+	/* Clear any bulk OUT halt condition */
+	if ( ( rc = usb_endpoint_clear_halt ( &usbblk->out ) ) != 0 ) {
+		DBGC ( usbblk, "USBBLK %s could not reset bulk OUT: %s\n",
+		       usbblk->func->name, strerror ( rc ) );
+		goto err_clear_out;
+	}
+
+	/* Open bulk IN endpoint */
+	if ( ( rc = usb_endpoint_open ( &usbblk->in ) ) != 0 ) {
+		DBGC ( usbblk, "USBBLK %s could not open bulk IN: %s\n",
+		       usbblk->func->name, strerror ( rc ) );
+		goto err_open_in;
+	}
+
+	/* Clear any bulk IN halt condition */
+	if ( ( rc = usb_endpoint_clear_halt ( &usbblk->in ) ) != 0 ) {
+		DBGC ( usbblk, "USBBLK %s could not reset bulk IN: %s\n",
+		       usbblk->func->name, strerror ( rc ) );
+		goto err_clear_in;
+	}
+
+	return 0;
+
+ err_clear_in:
+	usb_endpoint_close ( &usbblk->in );
+ err_open_in:
+ err_clear_out:
+	usb_endpoint_close ( &usbblk->out );
+ err_open_out:
+ err_reset:
+	return rc;
+}
+
+/**
+ * Close endpoints
+ *
+ * @v usbblk		USB block device
+ */
+static void usbblk_close ( struct usbblk_device *usbblk ) {
+
+	/* Close bulk OUT endpoint */
+	if ( usbblk->out.open )
+		usb_endpoint_close ( &usbblk->out );
+
+	/* Close bulk IN endpoint */
+	if ( usbblk->in.open )
+		usb_endpoint_close ( &usbblk->in );
+}
+
+/******************************************************************************
+ *
+ * Bulk OUT endpoint
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Issue bulk OUT command
+ *
+ * @v usbblk		USB block device
+ * @ret rc		Return status code
+ */
+static int usbblk_out_command ( struct usbblk_device *usbblk ) {
+	struct usbblk_command *cmd = &usbblk->cmd;
+	struct usbblk_command_wrapper *wrapper;
+	struct io_buffer *iobuf;
+	int rc;
+
+	/* Sanity checks */
+	assert ( cmd->tag );
+	assert ( ! ( cmd->scsi.data_in_len && cmd->scsi.data_out_len ) );
+
+	/* Allocate I/O buffer */
+	iobuf = alloc_iob ( sizeof ( *wrapper ) );
+	if ( ! iobuf ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Populate command */
+	wrapper = iob_put ( iobuf, sizeof ( *wrapper ) );
+	memset ( wrapper, 0, sizeof ( *wrapper ) );
+	wrapper->signature = cpu_to_le32 ( USBBLK_COMMAND_SIGNATURE );
+	wrapper->tag = cmd->tag; /* non-endian */
+	if ( cmd->scsi.data_out_len ) {
+		wrapper->len = cpu_to_le32 ( cmd->scsi.data_out_len );
+	} else {
+		wrapper->len = cpu_to_le32 ( cmd->scsi.data_in_len );
+		wrapper->flags = USB_DIR_IN;
+	}
+	wrapper->lun = ntohs ( cmd->scsi.lun.u16[0] );
+	wrapper->cblen = sizeof ( wrapper->cb );
+	memcpy ( wrapper->cb, &cmd->scsi.cdb, sizeof ( wrapper->cb ) );
+
+	/* Issue command */
+	if ( ( rc = usb_stream ( &usbblk->out, iobuf, 0 ) ) != 0 ) {
+		DBGC ( usbblk, "USBBLK %s bulk OUT could not issue command: "
+		       "%s\n", usbblk->func->name, strerror ( rc ) );
+		goto err_stream;
+	}
+
+	return 0;
+
+ err_stream:
+	free_iob ( iobuf );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Send bulk OUT data block
+ *
+ * @v usbblk		USB block device
+ * @ret rc		Return status code
+ */
+static int usbblk_out_data ( struct usbblk_device *usbblk ) {
+	struct usbblk_command *cmd = &usbblk->cmd;
+	struct io_buffer *iobuf;
+	size_t len;
+	int rc;
+
+	/* Calculate length */
+	assert ( cmd->tag );
+	assert ( cmd->scsi.data_out != UNULL );
+	assert ( cmd->offset < cmd->scsi.data_out_len );
+	len = ( cmd->scsi.data_out_len - cmd->offset );
+	if ( len > USBBLK_MAX_LEN )
+		len = USBBLK_MAX_LEN;
+	assert ( ( len % usbblk->out.mtu ) == 0 );
+
+	/* Allocate I/O buffer */
+	iobuf = alloc_iob ( len );
+	if ( ! iobuf ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Populate I/O buffer */
+	copy_from_user ( iob_put ( iobuf, len ), cmd->scsi.data_out,
+			 cmd->offset, len );
+
+	/* Send data */
+	if ( ( rc = usb_stream ( &usbblk->out, iobuf, 0 ) ) != 0 ) {
+		DBGC ( usbblk, "USBBLK %s bulk OUT could not send data: %s\n",
+		       usbblk->func->name, strerror ( rc ) );
+		goto err_stream;
+	}
+
+	/* Consume data */
+	cmd->offset += len;
+
+	return 0;
+
+ err_stream:
+	free_iob ( iobuf );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Refill bulk OUT endpoint
+ *
+ * @v usbblk		USB block device
+ * @ret rc		Return status code
+ */
+static int usbblk_out_refill ( struct usbblk_device *usbblk ) {
+	struct usbblk_command *cmd = &usbblk->cmd;
+	int rc;
+
+	/* Sanity checks */
+	assert ( cmd->tag );
+
+	/* Refill endpoint */
+	while ( ( cmd->offset < cmd->scsi.data_out_len ) &&
+		( usbblk->out.fill < USBBLK_MAX_FILL ) ) {
+		if ( ( rc = usbblk_out_data ( usbblk ) ) != 0 )
+			return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Complete bulk OUT transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void usbblk_out_complete ( struct usb_endpoint *ep,
+				  struct io_buffer *iobuf, int rc ) {
+	struct usbblk_device *usbblk =
+		container_of ( ep, struct usbblk_device, out );
+	struct usbblk_command *cmd = &usbblk->cmd;
+
+	/* Ignore cancellations after closing endpoint */
+	if ( ! ep->open )
+		goto drop;
+
+	/* Sanity check */
+	assert ( cmd->tag );
+
+	/* Check for failures */
+	if ( rc != 0 ) {
+		DBGC ( usbblk, "USBBLK %s bulk OUT failed: %s\n",
+		       usbblk->func->name, strerror ( rc ) );
+		goto err;
+	}
+
+	/* Trigger refill process, if applicable */
+	if ( cmd->offset < cmd->scsi.data_out_len )
+		process_add ( &usbblk->process );
+
+ drop:
+	/* Free I/O buffer */
+	free_iob ( iobuf );
+
+	return;
+
+ err:
+	free_iob ( iobuf );
+	usbblk_stop ( usbblk, rc );
+}
+
+/** Bulk OUT endpoint operations */
+static struct usb_endpoint_driver_operations usbblk_out_operations = {
+	.complete = usbblk_out_complete,
+};
+
+/******************************************************************************
+ *
+ * Bulk IN endpoint
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Handle bulk IN data block
+ *
+ * @v usbblk		USB block device
+ * @v data		Data block
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+static int usbblk_in_data ( struct usbblk_device *usbblk, const void *data,
+			    size_t len ) {
+	struct usbblk_command *cmd = &usbblk->cmd;
+
+	/* Sanity checks */
+	assert ( cmd->tag );
+	assert ( cmd->scsi.data_in != UNULL );
+	assert ( cmd->offset <= cmd->scsi.data_in_len );
+	assert ( len <= ( cmd->scsi.data_in_len - cmd->offset ) );
+
+	/* Store data */
+	copy_to_user ( cmd->scsi.data_in, cmd->offset, data, len );
+	cmd->offset += len;
+
+	return 0;
+}
+
+/**
+ * Handle bulk IN status
+ *
+ * @v usbblk		USB block device
+ * @v data		Status data
+ * @v len		Length of status data
+ * @ret rc		Return status code
+ */
+static int usbblk_in_status ( struct usbblk_device *usbblk, const void *data,
+			      size_t len ) {
+	struct usbblk_command *cmd = &usbblk->cmd;
+	const struct usbblk_status_wrapper *stat;
+
+	/* Sanity checks */
+	assert ( cmd->tag );
+
+	/* Validate length */
+	if ( len < sizeof ( *stat ) ) {
+		DBGC ( usbblk, "USBBLK %s bulk IN malformed status:\n",
+		       usbblk->func->name );
+		DBGC_HDA ( usbblk, 0, data, len );
+		return -EIO;
+	}
+	stat = data;
+
+	/* Validate signature */
+	if ( stat->signature != cpu_to_le32 ( USBBLK_STATUS_SIGNATURE ) ) {
+		DBGC ( usbblk, "USBBLK %s bulk IN invalid signature %08x:\n",
+		       usbblk->func->name, le32_to_cpu ( stat->signature ) );
+		DBGC_HDA ( usbblk, 0, stat, sizeof ( *stat ) );
+		return -EIO;
+	}
+
+	/* Validate tag */
+	if ( stat->tag != cmd->tag ) {
+		DBGC ( usbblk, "USBBLK %s bulk IN tag mismatch (got %08x, "
+		       "expected %08x):\n",
+		       usbblk->func->name, stat->tag, cmd->tag );
+		DBGC_HDA ( usbblk, 0, stat, sizeof ( *stat ) );
+		return -EIO;
+	}
+
+	/* Check status */
+	if ( stat->status ) {
+		DBGC ( usbblk, "USBBLK %s bulk IN status %02x:\n",
+		       usbblk->func->name, stat->status );
+		DBGC_HDA ( usbblk, 0, stat, sizeof ( *stat ) );
+		return -EIO;
+	}
+
+	/* Check for residual data */
+	if ( stat->residue ) {
+		DBGC ( usbblk, "USBBLK %s bulk IN residue %#x:\n",
+		       usbblk->func->name, le32_to_cpu ( stat->residue ) );
+		return -EIO;
+	}
+
+	/* Mark command as complete */
+	usbblk_stop ( usbblk, 0 );
+
+	return 0;
+}
+
+/**
+ * Refill bulk IN endpoint
+ *
+ * @v usbblk		USB block device
+ * @ret rc		Return status code
+ */
+static int usbblk_in_refill ( struct usbblk_device *usbblk ) {
+	struct usbblk_command *cmd = &usbblk->cmd;
+	struct usbblk_status_wrapper *stat;
+	size_t remaining;
+	unsigned int max;
+	int rc;
+
+	/* Sanity checks */
+	assert ( cmd->tag );
+
+	/* Calculate maximum required refill */
+	remaining = sizeof ( *stat );
+	if ( cmd->scsi.data_in_len ) {
+		assert ( cmd->offset <= cmd->scsi.data_in_len );
+		remaining += ( cmd->scsi.data_in_len - cmd->offset );
+	}
+	max = ( ( remaining + USBBLK_MAX_LEN - 1 ) / USBBLK_MAX_LEN );
+
+	/* Refill bulk IN endpoint */
+	if ( ( rc = usb_refill_limit ( &usbblk->in, max ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Complete bulk IN transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void usbblk_in_complete ( struct usb_endpoint *ep,
+				 struct io_buffer *iobuf, int rc ) {
+	struct usbblk_device *usbblk =
+		container_of ( ep, struct usbblk_device, in );
+	struct usbblk_command *cmd = &usbblk->cmd;
+	size_t remaining;
+	size_t len;
+
+	/* Ignore cancellations after closing endpoint */
+	if ( ! ep->open )
+		goto drop;
+
+	/* Sanity check */
+	assert ( cmd->tag );
+
+	/* Handle errors */
+	if ( rc != 0 ) {
+		DBGC ( usbblk, "USBBLK %s bulk IN failed: %s\n",
+		       usbblk->func->name, strerror ( rc ) );
+		goto err;
+	}
+
+	/* Trigger refill process */
+	process_add ( &usbblk->process );
+
+	/* Handle data portion, if any */
+	if ( cmd->scsi.data_in_len ) {
+		assert ( cmd->offset <= cmd->scsi.data_in_len );
+		remaining = ( cmd->scsi.data_in_len - cmd->offset );
+		len = iob_len ( iobuf );
+		if ( len > remaining )
+			len = remaining;
+		if ( len ) {
+			if ( ( rc = usbblk_in_data ( usbblk, iobuf->data,
+						     len ) ) != 0 )
+				goto err;
+			iob_pull ( iobuf, len );
+		}
+	}
+
+	/* Handle status portion, if any */
+	len = iob_len ( iobuf );
+	if ( len ) {
+		if ( ( rc = usbblk_in_status ( usbblk, iobuf->data,
+					       len ) ) != 0 )
+			goto err;
+	}
+
+ drop:
+	/* Free I/O buffer */
+	free_iob ( iobuf );
+
+	return;
+
+ err:
+	free_iob ( iobuf );
+	usbblk_stop ( usbblk, rc );
+}
+
+/** Bulk IN endpoint operations */
+static struct usb_endpoint_driver_operations usbblk_in_operations = {
+	.complete = usbblk_in_complete,
+};
+
+/******************************************************************************
+ *
+ * Refill process
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Refill endpoints
+ *
+ * @v usbblk		USB block device
+ */
+static void usbblk_step ( struct usbblk_device *usbblk ) {
+
+	/* Refill bulk OUT endpoint */
+	usbblk_out_refill ( usbblk );
+
+	/* Refill bulk IN endpoint */
+	usbblk_in_refill ( usbblk );
+}
+
+/** Refill process descriptor */
+static struct process_descriptor usbblk_process_desc =
+	PROC_DESC ( struct usbblk_device, process, usbblk_step );
+
+/******************************************************************************
+ *
+ * SCSI command management
+ *
+ ******************************************************************************
+ */
+
+/** Next command tag */
+static uint16_t usbblk_tag;
+
+/**
+ * Stop SCSI command
+ *
+ * @v usbblk		USB block device
+ * @v rc		Reason for stop
+ */
+static void usbblk_stop ( struct usbblk_device *usbblk, int rc ) {
+
+	/* Stop process */
+	process_del ( &usbblk->process );
+
+	/* Reset command */
+	memset ( &usbblk->cmd, 0, sizeof ( usbblk->cmd ) );
+
+	/* Close endpoints if an error occurred */
+	if ( rc != 0 ) {
+		DBGC ( usbblk, "USBBLK %s closing for error recovery\n",
+		       usbblk->func->name );
+		usbblk_close ( usbblk );
+	}
+
+	/* Terminate command */
+	intf_restart ( &usbblk->data, rc );
+}
+
+/**
+ * Start new SCSI command
+ *
+ * @v usbblk		USB block device
+ * @v scsicmd		SCSI command
+ * @ret rc		Return status code
+ */
+static int usbblk_start ( struct usbblk_device *usbblk,
+			  struct scsi_cmd *scsicmd ) {
+	struct usbblk_command *cmd = &usbblk->cmd;
+	int rc;
+
+	/* Fail if command is in progress */
+	if ( cmd->tag ) {
+		rc = -EBUSY;
+		DBGC ( usbblk, "USBBLK %s cannot support multiple commands\n",
+		       usbblk->func->name );
+		goto err_busy;
+	}
+
+	/* Refuse bidirectional commands */
+	if ( scsicmd->data_in_len && scsicmd->data_out_len ) {
+		rc = -EOPNOTSUPP;
+		DBGC ( usbblk, "USBBLK %s cannot support bidirectional "
+		       "commands\n", usbblk->func->name );
+		goto err_bidirectional;
+	}
+
+	/* Sanity checks */
+	assert ( ! process_running ( &usbblk->process ) );
+	assert ( cmd->offset == 0 );
+
+	/* Initialise command */
+	memcpy ( &cmd->scsi, scsicmd, sizeof ( cmd->scsi ) );
+	cmd->tag = ( USBBLK_TAG_MAGIC | ++usbblk_tag );
+
+	/* Issue bulk OUT command */
+	if ( ( rc = usbblk_out_command ( usbblk ) ) != 0 )
+		goto err_command;
+
+	/* Start refill process */
+	process_add ( &usbblk->process );
+
+	return 0;
+
+ err_command:
+	memset ( &usbblk->cmd, 0, sizeof ( usbblk->cmd ) );
+ err_bidirectional:
+ err_busy:
+	return rc;
+}
+
+/******************************************************************************
+ *
+ * SCSI interfaces
+ *
+ ******************************************************************************
+ */
+
+/** SCSI data interface operations */
+static struct interface_operation usbblk_data_operations[] = {
+	INTF_OP ( intf_close, struct usbblk_device *, usbblk_stop ),
+};
+
+/** SCSI data interface descriptor */
+static struct interface_descriptor usbblk_data_desc =
+	INTF_DESC ( struct usbblk_device, data, usbblk_data_operations );
+
+/**
+ * Check SCSI command flow-control window
+ *
+ * @v usbblk		USB block device
+ * @ret len		Length of window
+ */
+static size_t usbblk_scsi_window ( struct usbblk_device *usbblk ) {
+	struct usbblk_command *cmd = &usbblk->cmd;
+
+	/* Allow a single command if no command is currently in progress */
+	return ( cmd->tag ? 0 : 1 );
+}
+
+/**
+ * Issue SCSI command
+ *
+ * @v usbblk		USB block device
+ * @v data		SCSI data interface
+ * @v scsicmd		SCSI command
+ * @ret tag		Command tag, or negative error
+ */
+static int usbblk_scsi_command ( struct usbblk_device *usbblk,
+				 struct interface *data,
+				 struct scsi_cmd *scsicmd ) {
+	struct usbblk_command *cmd = &usbblk->cmd;
+	int rc;
+
+	/* (Re)open endpoints if needed */
+	if ( ( ! usbblk->in.open ) && ( ( rc = usbblk_open ( usbblk ) ) != 0 ) )
+		goto err_open;
+
+	/* Start new command */
+	if ( ( rc = usbblk_start ( usbblk, scsicmd ) ) != 0 )
+		goto err_start;
+
+	/* Attach to parent interface and return */
+	intf_plug_plug ( &usbblk->data, data );
+	return cmd->tag;
+
+	usbblk_stop ( usbblk, rc );
+ err_start:
+	usbblk_close ( usbblk );
+ err_open:
+	return rc;
+}
+
+/**
+ * Close SCSI interface
+ *
+ * @v usbblk		USB block device
+ * @v rc		Reason for close
+ */
+static void usbblk_scsi_close ( struct usbblk_device *usbblk, int rc ) {
+
+	/* Restart interfaces */
+	intfs_restart ( rc, &usbblk->scsi, &usbblk->data, NULL );
+
+	/* Stop any in-progress command */
+	usbblk_stop ( usbblk, rc );
+
+	/* Close endpoints */
+	usbblk_close ( usbblk );
+
+	/* Flag as closed */
+	usbblk->opened = 0;
+}
+
+/**
+ * Describe as an EFI device path
+ *
+ * @v usbblk		USB block device
+ * @ret path		EFI device path, or NULL on error
+ */
+static EFI_DEVICE_PATH_PROTOCOL *
+usbblk_efi_describe ( struct usbblk_device *usbblk ) {
+
+	return efi_usb_path ( usbblk->func );
+}
+
+/** SCSI command interface operations */
+static struct interface_operation usbblk_scsi_operations[] = {
+	INTF_OP ( scsi_command, struct usbblk_device *, usbblk_scsi_command ),
+	INTF_OP ( xfer_window, struct usbblk_device *, usbblk_scsi_window ),
+	INTF_OP ( intf_close, struct usbblk_device *, usbblk_scsi_close ),
+	EFI_INTF_OP ( efi_describe, struct usbblk_device *,
+		      usbblk_efi_describe ),
+};
+
+/** SCSI command interface descriptor */
+static struct interface_descriptor usbblk_scsi_desc =
+	INTF_DESC ( struct usbblk_device, scsi, usbblk_scsi_operations );
+
+/******************************************************************************
+ *
+ * SAN device interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Find USB block device
+ *
+ * @v name		USB block device name
+ * @ret usbblk		USB block device, or NULL
+ */
+static struct usbblk_device * usbblk_find ( const char *name ) {
+	struct usbblk_device *usbblk;
+
+	/* Look for matching device */
+	list_for_each_entry ( usbblk, &usbblk_devices, list ) {
+		if ( strcmp ( usbblk->func->name, name ) == 0 )
+			return usbblk;
+	}
+
+	return NULL;
+}
+
+/**
+ * Open USB block device URI
+ *
+ * @v parent		Parent interface
+ * @v uri		URI
+ * @ret rc		Return status code
+ */
+static int usbblk_open_uri ( struct interface *parent, struct uri *uri ) {
+	static struct scsi_lun lun;
+	struct usbblk_device *usbblk;
+	int rc;
+
+	/* Sanity check */
+	if ( ! uri->opaque )
+		return -EINVAL;
+
+	/* Find matching device */
+	usbblk = usbblk_find ( uri->opaque );
+	if ( ! usbblk )
+		return -ENOENT;
+
+	/* Fail if device is already open */
+	if ( usbblk->opened )
+		return -EBUSY;
+
+	/* Open SCSI device */
+	if ( ( rc = scsi_open ( parent, &usbblk->scsi, &lun ) ) != 0 ) {
+		DBGC ( usbblk, "USBBLK %s could not open SCSI device: %s\n",
+		       usbblk->func->name, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Mark as opened */
+	usbblk->opened = 1;
+
+	return 0;
+}
+
+/** USB block device URI opener */
+struct uri_opener usbblk_uri_opener __uri_opener = {
+	.scheme = "usb",
+	.open = usbblk_open_uri,
+};
+
+/******************************************************************************
+ *
+ * USB interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Probe device
+ *
+ * @v func		USB function
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+static int usbblk_probe ( struct usb_function *func,
+			  struct usb_configuration_descriptor *config ) {
+	struct usb_device *usb = func->usb;
+	struct usbblk_device *usbblk;
+	struct usb_interface_descriptor *desc;
+	int rc;
+
+	/* Allocate and initialise structure */
+	usbblk = zalloc ( sizeof ( *usbblk ) );
+	if ( ! usbblk ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	usbblk->func = func;
+	usb_endpoint_init ( &usbblk->out, usb, &usbblk_out_operations );
+	usb_endpoint_init ( &usbblk->in, usb, &usbblk_in_operations );
+	usb_refill_init ( &usbblk->in, 0, USBBLK_MAX_LEN, USBBLK_MAX_FILL );
+	intf_init ( &usbblk->scsi, &usbblk_scsi_desc, &usbblk->refcnt );
+	intf_init ( &usbblk->data, &usbblk_data_desc, &usbblk->refcnt );
+	process_init_stopped ( &usbblk->process, &usbblk_process_desc,
+			       &usbblk->refcnt );
+
+	/* Locate interface descriptor */
+	desc = usb_interface_descriptor ( config, func->interface[0], 0 );
+	if ( ! desc ) {
+		DBGC ( usbblk, "USBBLK %s missing interface descriptor\n",
+		       usbblk->func->name );
+		rc = -ENOENT;
+		goto err_desc;
+	}
+
+	/* Describe endpoints */
+	if ( ( rc = usb_endpoint_described ( &usbblk->out, config, desc,
+					     USB_BULK_OUT, 0 ) ) != 0 ) {
+		DBGC ( usbblk, "USBBLK %s could not describe bulk OUT: %s\n",
+		       usbblk->func->name, strerror ( rc ) );
+		goto err_out;
+	}
+	if ( ( rc = usb_endpoint_described ( &usbblk->in, config, desc,
+					     USB_BULK_IN, 0 ) ) != 0 ) {
+		DBGC ( usbblk, "USBBLK %s could not describe bulk IN: %s\n",
+		       usbblk->func->name, strerror ( rc ) );
+		goto err_in;
+	}
+
+	/* Add to list of devices */
+	list_add_tail ( &usbblk->list, &usbblk_devices );
+
+	usb_func_set_drvdata ( func, usbblk );
+	return 0;
+
+ err_in:
+ err_out:
+ err_desc:
+	ref_put ( &usbblk->refcnt );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove device
+ *
+ * @v func		USB function
+ */
+static void usbblk_remove ( struct usb_function *func ) {
+	struct usbblk_device *usbblk = usb_func_get_drvdata ( func );
+
+	/* Remove from list of devices */
+	list_del ( &usbblk->list );
+
+	/* Close all interfaces */
+	usbblk_scsi_close ( usbblk, -ENODEV );
+
+	/* Shut down interfaces */
+	intfs_shutdown ( -ENODEV, &usbblk->scsi, &usbblk->data, NULL );
+
+	/* Drop reference */
+	ref_put ( &usbblk->refcnt );
+}
+
+/** Mass storage class device IDs */
+static struct usb_device_id usbblk_ids[] = {
+	{
+		.name = "usbblk",
+		.vendor = USB_ANY_ID,
+		.product = USB_ANY_ID,
+	},
+};
+
+/** Mass storage driver */
+struct usb_driver usbblk_driver __usb_driver = {
+	.ids = usbblk_ids,
+	.id_count = ( sizeof ( usbblk_ids ) / sizeof ( usbblk_ids[0] ) ),
+	.class = USB_CLASS_ID ( USB_CLASS_MSC, USB_SUBCLASS_MSC_SCSI,
+				USB_PROTOCOL_MSC_BULK ),
+	.score = USB_SCORE_NORMAL,
+	.probe = usbblk_probe,
+	.remove = usbblk_remove,
+};
diff --git a/src/drivers/usb/usbblk.h b/src/drivers/usb/usbblk.h
new file mode 100644
index 000000000..65d0705e3
--- /dev/null
+++ b/src/drivers/usb/usbblk.h
@@ -0,0 +1,121 @@
+#ifndef _USBBLK_H
+#define _USBBLK_H
+
+/** @file
+ *
+ * USB mass storage driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/usb.h>
+#include <ipxe/scsi.h>
+#include <ipxe/interface.h>
+
+/** Mass storage class code */
+#define USB_CLASS_MSC 0x08
+
+/** SCSI command set subclass code */
+#define USB_SUBCLASS_MSC_SCSI 0x06
+
+/** Bulk-only transport protocol */
+#define USB_PROTOCOL_MSC_BULK 0x50
+
+/** Mass storage reset command */
+#define USBBLK_RESET ( USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE | \
+		       USB_REQUEST_TYPE ( 255 ) )
+
+/** Command block wrapper */
+struct usbblk_command_wrapper {
+	/** Signature */
+	uint32_t signature;
+	/** Tag */
+	uint32_t tag;
+	/** Data transfer length */
+	uint32_t len;
+	/** Flags */
+	uint8_t flags;
+	/** LUN */
+	uint8_t lun;
+	/** Command block length */
+	uint8_t cblen;
+	/** Command block */
+	uint8_t cb[16];
+} __attribute__ (( packed ));
+
+/** Command block wrapper signature */
+#define USBBLK_COMMAND_SIGNATURE 0x43425355UL
+
+/** Command status wrapper */
+struct usbblk_status_wrapper {
+	/** Signature */
+	uint32_t signature;
+	/** Tag */
+	uint32_t tag;
+	/** Data residue */
+	uint32_t residue;
+	/** Status */
+	uint8_t status;
+} __attribute__ (( packed ));
+
+/** Command status wrapper signature */
+#define USBBLK_STATUS_SIGNATURE 0x53425355UL
+
+/** A USB mass storage command */
+struct usbblk_command {
+	/** SCSI command */
+	struct scsi_cmd scsi;
+	/** Command tag (0 for no command in progress) */
+	uint32_t tag;
+	/** Offset within data buffer */
+	size_t offset;
+};
+
+/** A USB mass storage device */
+struct usbblk_device {
+	/** Reference count */
+	struct refcnt refcnt;
+	/** List of devices */
+	struct list_head list;
+
+	/** USB function */
+	struct usb_function *func;
+	/** Bulk OUT endpoint */
+	struct usb_endpoint out;
+	/** Bulk IN endpoint */
+	struct usb_endpoint in;
+
+	/** SCSI command-issuing interface */
+	struct interface scsi;
+	/** SCSI data interface */
+	struct interface data;
+	/** Command process */
+	struct process process;
+	/** Device opened flag */
+	int opened;
+
+	/** Current command (if any) */
+	struct usbblk_command cmd;
+};
+
+/** Command tag magic
+ *
+ * This is a policy decision.
+ */
+#define USBBLK_TAG_MAGIC 0x18ae0000
+
+/** Maximum length of USB data block
+ *
+ * This is a policy decision.
+ */
+#define USBBLK_MAX_LEN 2048
+
+/** Maximum endpoint fill level
+ *
+ * This is a policy decision.
+ */
+#define USBBLK_MAX_FILL 4
+
+#endif /* _USBBLK_H */
diff --git a/src/drivers/usb/usbio.c b/src/drivers/usb/usbio.c
index dfb93dab1..278b43cd3 100644
--- a/src/drivers/usb/usbio.c
+++ b/src/drivers/usb/usbio.c
@@ -29,6 +29,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <assert.h>
 #include <ipxe/efi/efi.h>
 #include <ipxe/efi/efi_driver.h>
+#include <ipxe/efi/efi_path.h>
 #include <ipxe/efi/efi_utils.h>
 #include <ipxe/efi/Protocol/UsbIo.h>
 #include <ipxe/usb.h>
@@ -206,7 +207,7 @@ static int usbio_open ( struct usbio_device *usbio, unsigned int interface ) {
 	path = usbio->path;
 	usbpath = usbio->usbpath;
 	usbpath->InterfaceNumber = interface;
-	end = efi_devpath_end ( path );
+	end = efi_path_end ( path );
 
 	/* Locate handle for this endpoint's interface */
 	if ( ( efirc = bs->LocateDevicePath ( &efi_usb_io_protocol_guid, &path,
@@ -1503,7 +1504,7 @@ static int usbio_path ( struct usbio_device *usbio ) {
 	path = u.interface;
 
 	/* Locate end of device path and sanity check */
-	len = efi_devpath_len ( path );
+	len = efi_path_len ( path );
 	if ( len < sizeof ( *usbpath ) ) {
 		DBGC ( usbio, "USBIO %s underlength device path\n",
 		       efi_handle_name ( handle ) );
diff --git a/src/drivers/usb/xhci.c b/src/drivers/usb/xhci.c
index aa6ca73c4..cc48af033 100644
--- a/src/drivers/usb/xhci.c
+++ b/src/drivers/usb/xhci.c
@@ -31,7 +31,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <errno.h>
 #include <byteswap.h>
 #include <ipxe/malloc.h>
-#include <ipxe/umalloc.h>
 #include <ipxe/pci.h>
 #include <ipxe/usb.h>
 #include <ipxe/init.h>
@@ -294,9 +293,9 @@ static void xhci_init ( struct xhci_device *xhci, void *regs ) {
 
 	/* Read structural parameters 2 */
 	hcsparams2 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS2 );
-	xhci->scratchpads = XHCI_HCSPARAMS2_SCRATCHPADS ( hcsparams2 );
+	xhci->scratch.count = XHCI_HCSPARAMS2_SCRATCHPADS ( hcsparams2 );
 	DBGC2 ( xhci, "XHCI %s needs %d scratchpads\n",
-		xhci->name, xhci->scratchpads );
+		xhci->name, xhci->scratch.count );
 
 	/* Read capability parameters 1 */
 	hccparams1 = readl ( xhci->cap + XHCI_CAP_HCCPARAMS1 );
@@ -918,27 +917,29 @@ static int xhci_dcbaa_alloc ( struct xhci_device *xhci ) {
 	 * align on its own size (rounded up to a power of two and
 	 * with a minimum of 64 bytes).
 	 */
-	len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa[0] ) );
-	xhci->dcbaa = malloc_dma ( len, xhci_align ( len ) );
-	if ( ! xhci->dcbaa ) {
+	len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa.context[0] ) );
+	xhci->dcbaa.context = dma_alloc ( xhci->dma, &xhci->dcbaa.map, len,
+					  xhci_align ( len ) );
+	if ( ! xhci->dcbaa.context ) {
 		DBGC ( xhci, "XHCI %s could not allocate DCBAA\n", xhci->name );
 		rc = -ENOMEM;
 		goto err_alloc;
 	}
-	memset ( xhci->dcbaa, 0, len );
+	memset ( xhci->dcbaa.context, 0, len );
 
 	/* Program DCBAA pointer */
-	dcbaap = virt_to_phys ( xhci->dcbaa );
+	dcbaap = dma ( &xhci->dcbaa.map, xhci->dcbaa.context );
 	if ( ( rc = xhci_writeq ( xhci, dcbaap,
 				  xhci->op + XHCI_OP_DCBAAP ) ) != 0 )
 		goto err_writeq;
 
-	DBGC2 ( xhci, "XHCI %s DCBAA at [%08lx,%08lx)\n",
-		xhci->name, dcbaap, ( dcbaap + len ) );
+	DBGC2 ( xhci, "XHCI %s DCBAA at [%08lx,%08lx)\n", xhci->name,
+		virt_to_phys ( xhci->dcbaa.context ),
+		( virt_to_phys ( xhci->dcbaa.context ) + len ) );
 	return 0;
 
  err_writeq:
-	free_dma ( xhci->dcbaa, len );
+	dma_free ( &xhci->dcbaa.map, xhci->dcbaa.context, len );
  err_alloc:
 	return rc;
 }
@@ -954,14 +955,14 @@ static void xhci_dcbaa_free ( struct xhci_device *xhci ) {
 
 	/* Sanity check */
 	for ( i = 0 ; i <= xhci->slots ; i++ )
-		assert ( xhci->dcbaa[i] == 0 );
+		assert ( xhci->dcbaa.context[i] == 0 );
 
 	/* Clear DCBAA pointer */
 	xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_DCBAAP );
 
 	/* Free DCBAA */
-	len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa[0] ) );
-	free_dma ( xhci->dcbaa, len );
+	len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa.context[0] ) );
+	dma_free ( &xhci->dcbaa.map, xhci->dcbaa.context, len );
 }
 
 /******************************************************************************
@@ -978,32 +979,34 @@ static void xhci_dcbaa_free ( struct xhci_device *xhci ) {
  * @ret rc		Return status code
  */
 static int xhci_scratchpad_alloc ( struct xhci_device *xhci ) {
+	struct xhci_scratchpad *scratch = &xhci->scratch;
+	size_t buffer_len;
 	size_t array_len;
-	size_t len;
-	physaddr_t phys;
+	physaddr_t addr;
 	unsigned int i;
 	int rc;
 
 	/* Do nothing if no scratchpad buffers are used */
-	if ( ! xhci->scratchpads )
+	if ( ! scratch->count )
 		return 0;
 
-	/* Allocate scratchpads */
-	len = ( xhci->scratchpads * xhci->pagesize );
-	xhci->scratchpad = umalloc ( len );
-	if ( ! xhci->scratchpad ) {
+	/* Allocate scratchpad buffers */
+	buffer_len = ( scratch->count * xhci->pagesize );
+	scratch->buffer = dma_umalloc ( xhci->dma, &scratch->buffer_map,
+					buffer_len, xhci->pagesize );
+	if ( ! scratch->buffer ) {
 		DBGC ( xhci, "XHCI %s could not allocate scratchpad buffers\n",
 		       xhci->name );
 		rc = -ENOMEM;
 		goto err_alloc;
 	}
-	memset_user ( xhci->scratchpad, 0, 0, len );
+	memset_user ( scratch->buffer, 0, 0, buffer_len );
 
 	/* Allocate scratchpad array */
-	array_len = ( xhci->scratchpads * sizeof ( xhci->scratchpad_array[0] ));
-	xhci->scratchpad_array =
-		malloc_dma ( array_len, xhci_align ( array_len ) );
-	if ( ! xhci->scratchpad_array ) {
+	array_len = ( scratch->count * sizeof ( scratch->array[0] ) );
+	scratch->array = dma_alloc ( xhci->dma, &scratch->array_map,
+				     array_len, xhci_align ( array_len ) );
+	if ( ! scratch->array ) {
 		DBGC ( xhci, "XHCI %s could not allocate scratchpad buffer "
 		       "array\n", xhci->name );
 		rc = -ENOMEM;
@@ -1011,25 +1014,28 @@ static int xhci_scratchpad_alloc ( struct xhci_device *xhci ) {
 	}
 
 	/* Populate scratchpad array */
-	for ( i = 0 ; i < xhci->scratchpads ; i++ ) {
-		phys = user_to_phys ( xhci->scratchpad, ( i * xhci->pagesize ));
-		xhci->scratchpad_array[i] = phys;
+	addr = dma_phys ( &scratch->buffer_map,
+			  user_to_phys ( scratch->buffer, 0 ) );
+	for ( i = 0 ; i < scratch->count ; i++ ) {
+		scratch->array[i] = cpu_to_le64 ( addr );
+		addr += xhci->pagesize;
 	}
 
 	/* Set scratchpad array pointer */
-	assert ( xhci->dcbaa != NULL );
-	xhci->dcbaa[0] = cpu_to_le64 ( virt_to_phys ( xhci->scratchpad_array ));
+	assert ( xhci->dcbaa.context != NULL );
+	xhci->dcbaa.context[0] = cpu_to_le64 ( dma ( &scratch->array_map,
+						     scratch->array ) );
 
 	DBGC2 ( xhci, "XHCI %s scratchpad [%08lx,%08lx) array [%08lx,%08lx)\n",
-		xhci->name, user_to_phys ( xhci->scratchpad, 0 ),
-		user_to_phys ( xhci->scratchpad, len ),
-		virt_to_phys ( xhci->scratchpad_array ),
-		( virt_to_phys ( xhci->scratchpad_array ) + array_len ) );
+		xhci->name, user_to_phys ( scratch->buffer, 0 ),
+		user_to_phys ( scratch->buffer, buffer_len ),
+		virt_to_phys ( scratch->array ),
+		( virt_to_phys ( scratch->array ) + array_len ) );
 	return 0;
 
-	free_dma ( xhci->scratchpad_array, array_len );
+	dma_free ( &scratch->array_map, scratch->array, array_len );
  err_alloc_array:
-	ufree ( xhci->scratchpad );
+	dma_ufree ( &scratch->buffer_map, scratch->buffer, buffer_len );
  err_alloc:
 	return rc;
 }
@@ -1040,22 +1046,25 @@ static int xhci_scratchpad_alloc ( struct xhci_device *xhci ) {
  * @v xhci		xHCI device
  */
 static void xhci_scratchpad_free ( struct xhci_device *xhci ) {
+	struct xhci_scratchpad *scratch = &xhci->scratch;
 	size_t array_len;
+	size_t buffer_len;
 
 	/* Do nothing if no scratchpad buffers are used */
-	if ( ! xhci->scratchpads )
+	if ( ! scratch->count )
 		return;
 
 	/* Clear scratchpad array pointer */
-	assert ( xhci->dcbaa != NULL );
-	xhci->dcbaa[0] = 0;
+	assert ( xhci->dcbaa.context != NULL );
+	xhci->dcbaa.context[0] = 0;
 
 	/* Free scratchpad array */
-	array_len = ( xhci->scratchpads * sizeof ( xhci->scratchpad_array[0] ));
-	free_dma ( xhci->scratchpad_array, array_len );
+	array_len = ( scratch->count * sizeof ( scratch->array[0] ) );
+	dma_free ( &scratch->array_map, scratch->array, array_len );
 
-	/* Free scratchpads */
-	ufree ( xhci->scratchpad );
+	/* Free scratchpad buffers */
+	buffer_len = ( scratch->count * xhci->pagesize );
+	dma_ufree ( &scratch->buffer_map, scratch->buffer, buffer_len );
 }
 
 /******************************************************************************
@@ -1202,7 +1211,8 @@ static int xhci_ring_alloc ( struct xhci_device *xhci,
 	}
 
 	/* Allocate TRBs */
-	ring->trb = malloc_dma ( ring->len, xhci_align ( ring->len ) );
+	ring->trb = dma_alloc ( xhci->dma, &ring->map, ring->len,
+				xhci_align ( ring->len ) );
 	if ( ! ring->trb ) {
 		rc = -ENOMEM;
 		goto err_alloc_trb;
@@ -1211,14 +1221,14 @@ static int xhci_ring_alloc ( struct xhci_device *xhci,
 
 	/* Initialise Link TRB */
 	link = &ring->trb[count].link;
-	link->next = cpu_to_le64 ( virt_to_phys ( ring->trb ) );
+	link->next = cpu_to_le64 ( dma ( &ring->map, ring->trb ) );
 	link->flags = XHCI_TRB_TC;
 	link->type = XHCI_TRB_LINK;
 	ring->link = link;
 
 	return 0;
 
-	free_dma ( ring->trb, ring->len );
+	dma_free ( &ring->map, ring->trb, ring->len );
  err_alloc_trb:
 	free ( ring->iobuf );
  err_alloc_iobuf:
@@ -1256,7 +1266,7 @@ static void xhci_ring_free ( struct xhci_trb_ring *ring ) {
 		assert ( ring->iobuf[i] == NULL );
 
 	/* Free TRBs */
-	free_dma ( ring->trb, ring->len );
+	dma_free ( &ring->map, ring->trb, ring->len );
 
 	/* Free I/O buffers */
 	free ( ring->iobuf );
@@ -1422,13 +1432,14 @@ static int xhci_command_alloc ( struct xhci_device *xhci ) {
 		goto err_ring_alloc;
 
 	/* Program command ring control register */
-	crp = virt_to_phys ( xhci->command.trb );
+	crp = dma ( &xhci->command.map, xhci->command.trb );
 	if ( ( rc = xhci_writeq ( xhci, ( crp | XHCI_CRCR_RCS ),
 				  xhci->op + XHCI_OP_CRCR ) ) != 0 )
 		goto err_writeq;
 
-	DBGC2 ( xhci, "XHCI %s CRCR at [%08lx,%08lx)\n",
-		xhci->name, crp, ( crp + xhci->command.len ) );
+	DBGC2 ( xhci, "XHCI %s CRCR at [%08lx,%08lx)\n", xhci->name,
+		virt_to_phys ( xhci->command.trb ),
+		( virt_to_phys ( xhci->command.trb ) + xhci->command.len ) );
 	return 0;
 
  err_writeq:
@@ -1469,7 +1480,8 @@ static int xhci_event_alloc ( struct xhci_device *xhci ) {
 	/* Allocate event ring */
 	count = ( 1 << XHCI_EVENT_TRBS_LOG2 );
 	len = ( count * sizeof ( event->trb[0] ) );
-	event->trb = malloc_dma ( len, xhci_align ( len ) );
+	event->trb = dma_alloc ( xhci->dma, &event->trb_map, len,
+				 xhci_align ( len ) );
 	if ( ! event->trb ) {
 		rc = -ENOMEM;
 		goto err_alloc_trb;
@@ -1477,22 +1489,25 @@ static int xhci_event_alloc ( struct xhci_device *xhci ) {
 	memset ( event->trb, 0, len );
 
 	/* Allocate event ring segment table */
-	event->segment = malloc_dma ( sizeof ( event->segment[0] ),
-				      xhci_align ( sizeof (event->segment[0])));
+	event->segment = dma_alloc ( xhci->dma, &event->segment_map,
+				     sizeof ( event->segment[0] ),
+				     xhci_align ( sizeof (event->segment[0])));
 	if ( ! event->segment ) {
 		rc = -ENOMEM;
 		goto err_alloc_segment;
 	}
 	memset ( event->segment, 0, sizeof ( event->segment[0] ) );
-	event->segment[0].base = cpu_to_le64 ( virt_to_phys ( event->trb ) );
+	event->segment[0].base = cpu_to_le64 ( dma ( &event->trb_map,
+						     event->trb ) );
 	event->segment[0].count = cpu_to_le32 ( count );
 
 	/* Program event ring registers */
 	writel ( 1, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) );
-	if ( ( rc = xhci_writeq ( xhci, virt_to_phys ( event->trb ),
+	if ( ( rc = xhci_writeq ( xhci, dma ( &event->trb_map, event->trb ),
 				  xhci->run + XHCI_RUN_ERDP ( 0 ) ) ) != 0 )
 		goto err_writeq_erdp;
-	if ( ( rc = xhci_writeq ( xhci, virt_to_phys ( event->segment ),
+	if ( ( rc = xhci_writeq ( xhci,
+				  dma ( &event->segment_map, event->segment ),
 				  xhci->run + XHCI_RUN_ERSTBA ( 0 ) ) ) != 0 )
 		goto err_writeq_erstba;
 
@@ -1501,16 +1516,17 @@ static int xhci_event_alloc ( struct xhci_device *xhci ) {
 		( virt_to_phys ( event->trb ) + len ),
 		virt_to_phys ( event->segment ),
 		( virt_to_phys ( event->segment ) +
-		  sizeof (event->segment[0] ) ) );
+		  sizeof ( event->segment[0] ) ) );
 	return 0;
 
 	xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) );
  err_writeq_erstba:
 	xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) );
  err_writeq_erdp:
-	free_dma ( event->trb, len );
+	dma_free ( &event->segment_map, event->segment,
+		   sizeof ( event->segment[0] ) );
  err_alloc_segment:
-	free_dma ( event->segment, sizeof ( event->segment[0] ) );
+	dma_free ( &event->trb_map, event->trb, len );
  err_alloc_trb:
 	return rc;
 }
@@ -1531,12 +1547,13 @@ static void xhci_event_free ( struct xhci_device *xhci ) {
 	xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) );
 
 	/* Free event ring segment table */
-	free_dma ( event->segment, sizeof ( event->segment[0] ) );
+	dma_free ( &event->segment_map, event->segment,
+		   sizeof ( event->segment[0] ) );
 
 	/* Free event ring */
 	count = ( 1 << XHCI_EVENT_TRBS_LOG2 );
 	len = ( count * sizeof ( event->trb[0] ) );
-	free_dma ( event->trb, len );
+	dma_free ( &event->trb_map, event->trb, len );
 }
 
 /**
@@ -1577,6 +1594,9 @@ static void xhci_transfer ( struct xhci_device *xhci,
 	iobuf = xhci_dequeue_multi ( &endpoint->ring );
 	assert ( iobuf != NULL );
 
+	/* Unmap I/O buffer */
+	iob_unmap ( iobuf );
+
 	/* Check for errors */
 	if ( ! ( ( trb->code == XHCI_CMPLT_SUCCESS ) ||
 		 ( trb->code == XHCI_CMPLT_SHORT ) ) ) {
@@ -1745,7 +1765,7 @@ static void xhci_event_poll ( struct xhci_device *xhci ) {
 
 	/* Update dequeue pointer if applicable */
 	if ( consumed ) {
-		xhci_writeq ( xhci, virt_to_phys ( trb ),
+		xhci_writeq ( xhci, dma ( &event->trb_map, trb ),
 			      xhci->run + XHCI_RUN_ERDP ( 0 ) );
 		profile_stop ( &xhci_event_profiler );
 	}
@@ -1774,7 +1794,7 @@ static void xhci_abort ( struct xhci_device *xhci ) {
 
 	/* Reset the command ring control register */
 	xhci_ring_reset ( &xhci->command );
-	crp = virt_to_phys ( xhci->command.trb );
+	crp = dma ( &xhci->command.map, xhci->command.trb );
 	xhci_writeq ( xhci, ( crp | XHCI_CRCR_RCS ), xhci->op + XHCI_OP_CRCR );
 }
 
@@ -1793,6 +1813,13 @@ static int xhci_command ( struct xhci_device *xhci, union xhci_trb *trb ) {
 	unsigned int i;
 	int rc;
 
+	/* Sanity check */
+	if ( xhci->pending ) {
+		DBGC ( xhci, "XHCI %s command ring busy\n", xhci->name );
+		rc = -EBUSY;
+		goto err_pending;
+	}
+
 	/* Record the pending command */
 	xhci->pending = trb;
 
@@ -1835,6 +1862,7 @@ static int xhci_command ( struct xhci_device *xhci, union xhci_trb *trb ) {
 
  err_enqueue:
 	xhci->pending = NULL;
+ err_pending:
 	return rc;
 }
 
@@ -1855,9 +1883,13 @@ static inline int xhci_nop ( struct xhci_device *xhci ) {
 	nop->type = XHCI_TRB_NOP_CMD;
 
 	/* Issue command and wait for completion */
-	if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 )
+	if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
+		DBGC ( xhci, "XHCI %s NOP failed: %s\n",
+		       xhci->name, strerror ( rc ) );
 		return rc;
+	}
 
+	DBGC2 ( xhci, "XHCI %s NOP completed successfully\n", xhci->name );
 	return 0;
 }
 
@@ -1942,13 +1974,15 @@ static int xhci_context ( struct xhci_device *xhci, struct xhci_slot *slot,
 						void *input ) ) {
 	union xhci_trb trb;
 	struct xhci_trb_context *context = &trb.context;
+	struct dma_mapping map;
 	size_t len;
 	void *input;
 	int rc;
 
 	/* Allocate an input context */
+	memset ( &map, 0, sizeof ( map ) );
 	len = xhci_input_context_offset ( xhci, XHCI_CTX_END );
-	input = malloc_dma ( len, xhci_align ( len ) );
+	input = dma_alloc ( xhci->dma, &map, len, xhci_align ( len ) );
 	if ( ! input ) {
 		rc = -ENOMEM;
 		goto err_alloc;
@@ -1961,7 +1995,7 @@ static int xhci_context ( struct xhci_device *xhci, struct xhci_slot *slot,
 	/* Construct command */
 	memset ( context, 0, sizeof ( *context ) );
 	context->type = type;
-	context->input = cpu_to_le64 ( virt_to_phys ( input ) );
+	context->input = cpu_to_le64 ( dma ( &map, input ) );
 	context->slot = slot->id;
 
 	/* Issue command and wait for completion */
@@ -1969,7 +2003,7 @@ static int xhci_context ( struct xhci_device *xhci, struct xhci_slot *slot,
 		goto err_command;
 
  err_command:
-	free_dma ( input, len );
+	dma_free ( &map, input, len );
  err_alloc:
 	return rc;
 }
@@ -1986,6 +2020,7 @@ static void xhci_address_device_input ( struct xhci_device *xhci,
 					struct xhci_slot *slot,
 					struct xhci_endpoint *endpoint,
 					void *input ) {
+	struct xhci_trb_ring *ring = &endpoint->ring;
 	struct xhci_control_context *control_ctx;
 	struct xhci_slot_context *slot_ctx;
 	struct xhci_endpoint_context *ep_ctx;
@@ -2011,7 +2046,7 @@ static void xhci_address_device_input ( struct xhci_device *xhci,
 	ep_ctx->type = XHCI_EP_TYPE_CONTROL;
 	ep_ctx->burst = endpoint->ep->burst;
 	ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
-	ep_ctx->dequeue = cpu_to_le64 ( virt_to_phys ( endpoint->ring.trb ) |
+	ep_ctx->dequeue = cpu_to_le64 ( dma ( &ring->map, ring->trb ) |
 					XHCI_EP_DCS );
 	ep_ctx->trb_len = cpu_to_le16 ( XHCI_EP0_TRB_LEN );
 }
@@ -2032,15 +2067,18 @@ static inline int xhci_address_device ( struct xhci_device *xhci,
 	/* Assign device address */
 	if ( ( rc = xhci_context ( xhci, slot, slot->endpoint[XHCI_CTX_EP0],
 				   XHCI_TRB_ADDRESS_DEVICE,
-				   xhci_address_device_input ) ) != 0 )
+				   xhci_address_device_input ) ) != 0 ) {
+		DBGC ( xhci, "XHCI %s slot %d could not assign address: %s\n",
+		       xhci->name, slot->id, strerror ( rc ) );
 		return rc;
+	}
 
 	/* Get assigned address */
 	slot_ctx = ( slot->context +
 		     xhci_device_context_offset ( xhci, XHCI_CTX_SLOT ) );
 	usb->address = slot_ctx->address;
-	DBGC2 ( xhci, "XHCI %s assigned address %d to %s\n",
-		xhci->name, usb->address, usb->name );
+	DBGC2 ( xhci, "XHCI %s slot %d assigned address %d to %s\n",
+		xhci->name, slot->id, usb->address, usb->name );
 
 	return 0;
 }
@@ -2057,6 +2095,7 @@ static void xhci_configure_endpoint_input ( struct xhci_device *xhci,
 					    struct xhci_slot *slot,
 					    struct xhci_endpoint *endpoint,
 					    void *input ) {
+	struct xhci_trb_ring *ring = &endpoint->ring;
 	struct xhci_control_context *control_ctx;
 	struct xhci_slot_context *slot_ctx;
 	struct xhci_endpoint_context *ep_ctx;
@@ -2079,7 +2118,7 @@ static void xhci_configure_endpoint_input ( struct xhci_device *xhci,
 	ep_ctx->type = endpoint->type;
 	ep_ctx->burst = endpoint->ep->burst;
 	ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
-	ep_ctx->dequeue = cpu_to_le64 ( virt_to_phys ( endpoint->ring.trb ) |
+	ep_ctx->dequeue = cpu_to_le64 ( dma ( &ring->map, ring->trb ) |
 					XHCI_EP_DCS );
 	ep_ctx->trb_len = cpu_to_le16 ( endpoint->ep->mtu ); /* best guess */
 }
@@ -2100,8 +2139,11 @@ static inline int xhci_configure_endpoint ( struct xhci_device *xhci,
 	/* Configure endpoint */
 	if ( ( rc = xhci_context ( xhci, slot, endpoint,
 				   XHCI_TRB_CONFIGURE_ENDPOINT,
-				   xhci_configure_endpoint_input ) ) != 0 )
+				   xhci_configure_endpoint_input ) ) != 0 ) {
+		DBGC ( xhci, "XHCI %s slot %d ctx %d could not configure: %s\n",
+		       xhci->name, slot->id, endpoint->ctx, strerror ( rc ) );
 		return rc;
+	}
 
 	DBGC2 ( xhci, "XHCI %s slot %d ctx %d configured\n",
 		xhci->name, slot->id, endpoint->ctx );
@@ -2151,8 +2193,12 @@ static inline int xhci_deconfigure_endpoint ( struct xhci_device *xhci,
 	/* Deconfigure endpoint */
 	if ( ( rc = xhci_context ( xhci, slot, endpoint,
 				   XHCI_TRB_CONFIGURE_ENDPOINT,
-				   xhci_deconfigure_endpoint_input ) ) != 0 )
+				   xhci_deconfigure_endpoint_input ) ) != 0 ) {
+		DBGC ( xhci, "XHCI %s slot %d ctx %d could not deconfigure: "
+		       "%s\n", xhci->name, slot->id, endpoint->ctx,
+		       strerror ( rc ) );
 		return rc;
+	}
 
 	DBGC2 ( xhci, "XHCI %s slot %d ctx %d deconfigured\n",
 		xhci->name, slot->id, endpoint->ctx );
@@ -2206,8 +2252,12 @@ static inline int xhci_evaluate_context ( struct xhci_device *xhci,
 	/* Configure endpoint */
 	if ( ( rc = xhci_context ( xhci, slot, endpoint,
 				   XHCI_TRB_EVALUATE_CONTEXT,
-				   xhci_evaluate_context_input ) ) != 0 )
+				   xhci_evaluate_context_input ) ) != 0 ) {
+		DBGC ( xhci, "XHCI %s slot %d ctx %d could not (re-)evaluate: "
+		       "%s\n", xhci->name, slot->id, endpoint->ctx,
+		       strerror ( rc ) );
 		return rc;
+	}
 
 	DBGC2 ( xhci, "XHCI %s slot %d ctx %d (re-)evaluated\n",
 		xhci->name, slot->id, endpoint->ctx );
@@ -2297,6 +2347,7 @@ xhci_set_tr_dequeue_pointer ( struct xhci_device *xhci,
 	unsigned int mask;
 	unsigned int index;
 	unsigned int dcs;
+	physaddr_t addr;
 	int rc;
 
 	/* Construct command */
@@ -2305,8 +2356,8 @@ xhci_set_tr_dequeue_pointer ( struct xhci_device *xhci,
 	mask = ring->mask;
 	dcs = ( ( ~( cons >> ring->shift ) ) & XHCI_EP_DCS );
 	index = ( cons & mask );
-	dequeue->dequeue =
-		cpu_to_le64 ( virt_to_phys ( &ring->trb[index] ) | dcs );
+	addr = dma ( &ring->map, &ring->trb[index] );
+	dequeue->dequeue = cpu_to_le64 ( addr | dcs );
 	dequeue->slot = slot->id;
 	dequeue->endpoint = endpoint->ctx;
 	dequeue->type = XHCI_TRB_SET_TR_DEQUEUE_POINTER;
@@ -2425,6 +2476,7 @@ static void xhci_endpoint_close ( struct usb_endpoint *ep ) {
 	/* Cancel any incomplete transfers */
 	while ( xhci_ring_fill ( &endpoint->ring ) ) {
 		iobuf = xhci_dequeue_multi ( &endpoint->ring );
+		iob_unmap ( iobuf );
 		usb_complete_err ( ep, iobuf, -ECANCELED );
 	}
 
@@ -2491,6 +2543,7 @@ static int xhci_endpoint_mtu ( struct usb_endpoint *ep ) {
 static int xhci_endpoint_message ( struct usb_endpoint *ep,
 				   struct io_buffer *iobuf ) {
 	struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct xhci_device *xhci = endpoint->xhci;
 	struct usb_setup_packet *packet;
 	unsigned int input;
 	size_t len;
@@ -2520,10 +2573,15 @@ static int xhci_endpoint_message ( struct usb_endpoint *ep,
 	if ( len )
 		setup->direction = ( input ? XHCI_SETUP_IN : XHCI_SETUP_OUT );
 
+	/* Map I/O buffer */
+	if ( ( rc = iob_map ( iobuf, xhci->dma, len,
+			      ( input ? DMA_RX : DMA_TX ) ) ) != 0 )
+		goto err_map;
+
 	/* Construct data stage TRB, if applicable */
 	if ( len ) {
 		data = &(trb++)->data;
-		data->data = cpu_to_le64 ( virt_to_phys ( iobuf->data ) );
+		data->data = cpu_to_le64 ( iob_dma ( iobuf ) );
 		data->len = cpu_to_le32 ( len );
 		data->type = XHCI_TRB_DATA;
 		data->direction = ( input ? XHCI_DATA_IN : XHCI_DATA_OUT );
@@ -2539,13 +2597,18 @@ static int xhci_endpoint_message ( struct usb_endpoint *ep,
 	/* Enqueue TRBs */
 	if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs,
 					 ( trb - trbs ) ) ) != 0 )
-		return rc;
+		goto err_enqueue;
 
 	/* Ring the doorbell */
 	xhci_doorbell ( &endpoint->ring );
 
 	profile_stop ( &xhci_message_profiler );
 	return 0;
+
+ err_enqueue:
+	iob_unmap ( iobuf );
+ err_map:
+	return rc;
 }
 
 /**
@@ -2579,12 +2642,13 @@ static unsigned int xhci_endpoint_count ( size_t len, int zlp ) {
 static int xhci_endpoint_stream ( struct usb_endpoint *ep,
 				  struct io_buffer *iobuf, int zlp ) {
 	struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
-	void *data = iobuf->data;
+	struct xhci_device *xhci = endpoint->xhci;
 	size_t len = iob_len ( iobuf );
 	unsigned int count = xhci_endpoint_count ( len, zlp );
 	union xhci_trb trbs[count];
 	union xhci_trb *trb = trbs;
 	struct xhci_trb_normal *normal;
+	physaddr_t data;
 	unsigned int i;
 	size_t trb_len;
 	int rc;
@@ -2592,6 +2656,13 @@ static int xhci_endpoint_stream ( struct usb_endpoint *ep,
 	/* Profile stream transfers */
 	profile_start ( &xhci_stream_profiler );
 
+	/* Map I/O buffer */
+	if ( ( rc = iob_map ( iobuf, xhci->dma, len,
+			      ( ( ep->address & USB_DIR_IN ) ?
+				DMA_RX : DMA_TX ) ) ) != 0 )
+		goto err_map;
+	data = iob_dma ( iobuf );
+
 	/* Construct normal TRBs */
 	memset ( &trbs, 0, sizeof ( trbs ) );
 	for ( i = 0 ; i < count ; i ++ ) {
@@ -2603,7 +2674,7 @@ static int xhci_endpoint_stream ( struct usb_endpoint *ep,
 
 		/* Construct normal TRB */
 		normal = &trb->normal;
-		normal->data = cpu_to_le64 ( virt_to_phys ( data ) );
+		normal->data = cpu_to_le64 ( data );
 		normal->len = cpu_to_le32 ( trb_len );
 		normal->type = XHCI_TRB_NORMAL;
 		normal->flags = XHCI_TRB_CH;
@@ -2624,13 +2695,18 @@ static int xhci_endpoint_stream ( struct usb_endpoint *ep,
 	/* Enqueue TRBs */
 	if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs,
 					 count ) ) != 0 )
-		return rc;
+		goto err_enqueue;
 
 	/* Ring the doorbell */
 	xhci_doorbell ( &endpoint->ring );
 
 	profile_stop ( &xhci_stream_profiler );
 	return 0;
+
+ err_enqueue:
+	iob_unmap ( iobuf );
+ err_map:
+	return rc;
 }
 
 /******************************************************************************
@@ -2693,7 +2769,8 @@ static int xhci_device_open ( struct usb_device *usb ) {
 
 	/* Allocate a device context */
 	len = xhci_device_context_offset ( xhci, XHCI_CTX_END );
-	slot->context = malloc_dma ( len, xhci_align ( len ) );
+	slot->context = dma_alloc ( xhci->dma, &slot->map, len,
+				    xhci_align ( len ) );
 	if ( ! slot->context ) {
 		rc = -ENOMEM;
 		goto err_alloc_context;
@@ -2701,16 +2778,17 @@ static int xhci_device_open ( struct usb_device *usb ) {
 	memset ( slot->context, 0, len );
 
 	/* Set device context base address */
-	assert ( xhci->dcbaa[id] == 0 );
-	xhci->dcbaa[id] = cpu_to_le64 ( virt_to_phys ( slot->context ) );
+	assert ( xhci->dcbaa.context[id] == 0 );
+	xhci->dcbaa.context[id] = cpu_to_le64 ( dma ( &slot->map,
+						      slot->context ) );
 
 	DBGC2 ( xhci, "XHCI %s slot %d device context [%08lx,%08lx) for %s\n",
 		xhci->name, slot->id, virt_to_phys ( slot->context ),
 		( virt_to_phys ( slot->context ) + len ), usb->name );
 	return 0;
 
-	xhci->dcbaa[id] = 0;
-	free_dma ( slot->context, len );
+	xhci->dcbaa.context[id] = 0;
+	dma_free ( &slot->map, slot->context, len );
  err_alloc_context:
 	xhci->slot[id] = NULL;
 	free ( slot );
@@ -2750,8 +2828,8 @@ static void xhci_device_close ( struct usb_device *usb ) {
 
 	/* Free slot */
 	if ( slot->context ) {
-		free_dma ( slot->context, len );
-		xhci->dcbaa[id] = 0;
+		dma_free ( &slot->map, slot->context, len );
+		xhci->dcbaa.context[id] = 0;
 	}
 	xhci->slot[id] = NULL;
 	free ( slot );
@@ -3261,7 +3339,7 @@ static int xhci_probe ( struct pci_device *pci ) {
 	/* Map registers */
 	bar_start = pci_bar_start ( pci, XHCI_BAR );
 	bar_size = pci_bar_size ( pci, XHCI_BAR );
-	xhci->regs = ioremap ( bar_start, bar_size );
+	xhci->regs = pci_ioremap ( pci, bar_start, bar_size );
 	if ( ! xhci->regs ) {
 		rc = -ENODEV;
 		goto err_ioremap;
@@ -3270,6 +3348,11 @@ static int xhci_probe ( struct pci_device *pci ) {
 	/* Initialise xHCI device */
 	xhci_init ( xhci, xhci->regs );
 
+	/* Configure DMA device */
+	xhci->dma = &pci->dma;
+	if ( xhci->addr64 )
+		dma_set_mask_64bit ( xhci->dma );
+
 	/* Initialise USB legacy support and claim ownership */
 	xhci_legacy_init ( xhci );
 	xhci_legacy_claim ( xhci );
diff --git a/src/drivers/usb/xhci.h b/src/drivers/usb/xhci.h
index e996363ea..6e02d70e0 100644
--- a/src/drivers/usb/xhci.h
+++ b/src/drivers/usb/xhci.h
@@ -854,6 +854,8 @@ struct xhci_trb_ring {
 	union xhci_trb *trb;
 	/** Length of transfer request blocks */
 	size_t len;
+	/** DMA mapping */
+	struct dma_mapping map;
 	/** Link TRB (if applicable) */
 	struct xhci_trb_link *link;
 
@@ -869,8 +871,12 @@ struct xhci_event_ring {
 	unsigned int cons;
 	/** Event ring segment table */
 	struct xhci_event_ring_segment *segment;
+	/** Event ring segment table DMA mapping */
+	struct dma_mapping segment_map;
 	/** Transfer request blocks */
 	union xhci_trb *trb;
+	/** Transfer request blocks DMA mapping */
+	struct dma_mapping trb_map;
 };
 
 /**
@@ -1035,10 +1041,34 @@ struct xhci_pch {
 /** Invalid protocol speed ID values quirk */
 #define XHCI_BAD_PSIV 0x0002
 
+/** Device context base address array */
+struct xhci_dcbaa {
+	/** Context base addresses */
+	uint64_t *context;
+	/** DMA mapping */
+	struct dma_mapping map;
+};
+
+/** Scratchpad buffer */
+struct xhci_scratchpad {
+	/** Number of page-sized scratchpad buffers */
+	unsigned int count;
+	/** Scratchpad buffer area */
+	userptr_t buffer;
+	/** Buffer DMA mapping */
+	struct dma_mapping buffer_map;
+	/** Scratchpad array */
+	uint64_t *array;
+	/** Array DMA mapping */
+	struct dma_mapping array_map;
+};
+
 /** An xHCI device */
 struct xhci_device {
 	/** Registers */
 	void *regs;
+	/** DMA device */
+	struct dma_device *dma;
 	/** Name */
 	const char *name;
 	/** Quirks */
@@ -1060,9 +1090,6 @@ struct xhci_device {
 	/** Number of ports */
 	unsigned int ports;
 
-	/** Number of page-sized scratchpad buffers */
-	unsigned int scratchpads;
-
 	/** 64-bit addressing capability */
 	int addr64;
 	/** Context size shift */
@@ -1077,12 +1104,10 @@ struct xhci_device {
 	unsigned int legacy;
 
 	/** Device context base address array */
-	uint64_t *dcbaa;
+	struct xhci_dcbaa dcbaa;
 
-	/** Scratchpad buffer area */
-	userptr_t scratchpad;
-	/** Scratchpad buffer array */
-	uint64_t *scratchpad_array;
+	/** Scratchpad buffer */
+	struct xhci_scratchpad scratch;
 
 	/** Command ring */
 	struct xhci_trb_ring command;
@@ -1111,6 +1136,8 @@ struct xhci_slot {
 	unsigned int id;
 	/** Slot context */
 	struct xhci_slot_context *context;
+	/** DMA mapping */
+	struct dma_mapping map;
 	/** Route string */
 	unsigned int route;
 	/** Root hub port number */