diff options
Diffstat (limited to 'src/drivers/net/gve.c')
| -rw-r--r-- | src/drivers/net/gve.c | 2016 |
1 files changed, 2016 insertions, 0 deletions
diff --git a/src/drivers/net/gve.c b/src/drivers/net/gve.c new file mode 100644 index 000000000..77eb4b674 --- /dev/null +++ b/src/drivers/net/gve.c @@ -0,0 +1,2016 @@ +/* + * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include <stdint.h> +#include <string.h> +#include <stdio.h> +#include <unistd.h> +#include <errno.h> +#include <assert.h> +#include <byteswap.h> +#include <ipxe/netdevice.h> +#include <ipxe/ethernet.h> +#include <ipxe/if_ether.h> +#include <ipxe/iobuf.h> +#include <ipxe/dma.h> +#include <ipxe/pci.h> +#include <ipxe/fault.h> +#include "gve.h" + +/** @file + * + * Google Virtual Ethernet network driver + * + */ + +/* Disambiguate the various error causes */ +#define EINFO_EIO_ADMIN_UNSET \ + __einfo_uniqify ( EINFO_EIO, 0x00, "Uncompleted" ) +#define EIO_ADMIN_UNSET \ + __einfo_error ( EINFO_EIO_ADMIN_UNSET ) +#define EINFO_EIO_ADMIN_ABORTED \ + __einfo_uniqify ( EINFO_EIO, 0x10, "Aborted" ) +#define EIO_ADMIN_ABORTED \ + __einfo_error ( EINFO_EIO_ADMIN_ABORTED ) +#define EINFO_EIO_ADMIN_EXISTS \ + __einfo_uniqify ( EINFO_EIO, 0x11, "Already exists" ) +#define EIO_ADMIN_EXISTS \ + __einfo_error ( EINFO_EIO_ADMIN_EXISTS ) +#define EINFO_EIO_ADMIN_CANCELLED \ + __einfo_uniqify ( EINFO_EIO, 0x12, "Cancelled" ) +#define EIO_ADMIN_CANCELLED \ + __einfo_error ( EINFO_EIO_ADMIN_CANCELLED ) +#define EINFO_EIO_ADMIN_DATALOSS \ + __einfo_uniqify ( EINFO_EIO, 0x13, "Data loss" ) +#define EIO_ADMIN_DATALOSS \ + __einfo_error ( EINFO_EIO_ADMIN_DATALOSS ) +#define EINFO_EIO_ADMIN_DEADLINE \ + __einfo_uniqify ( EINFO_EIO, 0x14, "Deadline exceeded" ) +#define EIO_ADMIN_DEADLINE \ + __einfo_error ( EINFO_EIO_ADMIN_DEADLINE ) +#define EINFO_EIO_ADMIN_PRECONDITION \ + __einfo_uniqify ( EINFO_EIO, 0x15, "Failed precondition" ) +#define EIO_ADMIN_PRECONDITION \ + __einfo_error ( EINFO_EIO_ADMIN_PRECONDITION ) +#define EINFO_EIO_ADMIN_INTERNAL \ + __einfo_uniqify ( EINFO_EIO, 0x16, "Internal error" ) +#define EIO_ADMIN_INTERNAL \ + __einfo_error ( EINFO_EIO_ADMIN_INTERNAL ) +#define EINFO_EIO_ADMIN_INVAL \ + __einfo_uniqify ( EINFO_EIO, 0x17, "Invalid argument" ) +#define EIO_ADMIN_INVAL \ + __einfo_error ( EINFO_EIO_ADMIN_INVAL ) +#define EINFO_EIO_ADMIN_NOT_FOUND \ + __einfo_uniqify ( EINFO_EIO, 0x18, "Not found" ) +#define EIO_ADMIN_NOT_FOUND \ + __einfo_error ( EINFO_EIO_ADMIN_NOT_FOUND ) +#define EINFO_EIO_ADMIN_RANGE \ + __einfo_uniqify ( EINFO_EIO, 0x19, "Out of range" ) +#define EIO_ADMIN_RANGE \ + __einfo_error ( EINFO_EIO_ADMIN_RANGE ) +#define EINFO_EIO_ADMIN_PERM \ + __einfo_uniqify ( EINFO_EIO, 0x1a, "Permission denied" ) +#define EIO_ADMIN_PERM \ + __einfo_error ( EINFO_EIO_ADMIN_PERM ) +#define EINFO_EIO_ADMIN_UNAUTH \ + __einfo_uniqify ( EINFO_EIO, 0x1b, "Unauthenticated" ) +#define EIO_ADMIN_UNAUTH \ + __einfo_error ( EINFO_EIO_ADMIN_UNAUTH ) +#define EINFO_EIO_ADMIN_RESOURCE \ + __einfo_uniqify ( EINFO_EIO, 0x1c, "Resource exhausted" ) +#define EIO_ADMIN_RESOURCE \ + __einfo_error ( EINFO_EIO_ADMIN_RESOURCE ) +#define EINFO_EIO_ADMIN_UNAVAIL \ + __einfo_uniqify ( EINFO_EIO, 0x1d, "Unavailable" ) +#define EIO_ADMIN_UNAVAIL \ + __einfo_error ( EINFO_EIO_ADMIN_UNAVAIL ) +#define EINFO_EIO_ADMIN_NOTSUP \ + __einfo_uniqify ( EINFO_EIO, 0x1e, "Unimplemented" ) +#define EIO_ADMIN_NOTSUP \ + __einfo_error ( EINFO_EIO_ADMIN_NOTSUP ) +#define EINFO_EIO_ADMIN_UNKNOWN \ + __einfo_uniqify ( EINFO_EIO, 0x1f, "Unknown error" ) +#define EIO_ADMIN_UNKNOWN \ + __einfo_error ( EINFO_EIO_ADMIN_UNKNOWN ) +#define EIO_ADMIN( status ) \ + EUNIQ ( EINFO_EIO, ( (status) & 0x1f ), \ + EIO_ADMIN_UNSET, EIO_ADMIN_ABORTED, EIO_ADMIN_EXISTS, \ + EIO_ADMIN_CANCELLED, EIO_ADMIN_DATALOSS, \ + EIO_ADMIN_DEADLINE, EIO_ADMIN_PRECONDITION, \ + EIO_ADMIN_INTERNAL, EIO_ADMIN_INVAL, \ + EIO_ADMIN_NOT_FOUND, EIO_ADMIN_RANGE, EIO_ADMIN_PERM, \ + EIO_ADMIN_UNAUTH, EIO_ADMIN_RESOURCE, \ + EIO_ADMIN_UNAVAIL, EIO_ADMIN_NOTSUP, EIO_ADMIN_UNKNOWN ) + +/****************************************************************************** + * + * Buffer layout + * + ****************************************************************************** + */ + +/** + * Get buffer offset (within queue page list allocation) + * + * @v queue Descriptor queue + * @v tag Buffer tag + * @ret addr Buffer address within queue page list address space + */ +static inline __attribute__ (( always_inline)) size_t +gve_offset ( struct gve_queue *queue, unsigned int tag ) { + + /* We allocate sufficient pages for the maximum fill level of + * buffers, and reuse the buffers in strict rotation as they + * are released by the hardware. + */ + assert ( tag < queue->fill ); + return ( tag * GVE_BUF_SIZE ); +} + +/** + * Get buffer address (within queue page list address space) + * + * @v queue Descriptor queue + * @v tag Buffer tag + * @ret addr Buffer address within queue page list address space + */ +static inline __attribute__ (( always_inline)) physaddr_t +gve_address ( struct gve_queue *queue, unsigned int tag ) { + + /* Pages are allocated as a single contiguous block */ + return ( queue->qpl.base + gve_offset ( queue, tag ) ); +} + +/** + * Get buffer address + * + * @v queue Descriptor queue + * @v tag Buffer tag + * @ret addr Buffer address + */ +static inline __attribute__ (( always_inline )) void * +gve_buffer ( struct gve_queue *queue, unsigned int tag ) { + + /* Pages are allocated as a single contiguous block */ + return ( queue->qpl.data + gve_offset ( queue, tag ) ); +} + +/****************************************************************************** + * + * Device reset + * + ****************************************************************************** + */ + +/** + * Reset hardware + * + * @v gve GVE device + * @ret rc Return status code + */ +static int gve_reset ( struct gve_nic *gve ) { + uint32_t pfn; + unsigned int i; + + /* Skip reset if admin queue page frame number is already + * clear. Triggering a reset on an already-reset device seems + * to cause a delayed reset to be scheduled. This can cause + * the device to end up in a reset loop, where each attempt to + * recover from reset triggers another reset a few seconds + * later. + */ + pfn = readl ( gve->cfg + GVE_CFG_ADMIN_PFN ); + if ( ! pfn ) { + DBGC ( gve, "GVE %p skipping reset\n", gve ); + return 0; + } + + /* Clear admin queue page frame number */ + writel ( 0, gve->cfg + GVE_CFG_ADMIN_PFN ); + wmb(); + + /* Wait for device to reset */ + for ( i = 0 ; i < GVE_RESET_MAX_WAIT_MS ; i++ ) { + + /* Delay */ + mdelay ( 1 ); + + /* Check for reset completion */ + pfn = readl ( gve->cfg + GVE_CFG_ADMIN_PFN ); + if ( ! pfn ) + return 0; + } + + DBGC ( gve, "GVE %p reset timed out (PFN %#08x devstat %#08x)\n", + gve, bswap_32 ( pfn ), + bswap_32 ( readl ( gve->cfg + GVE_CFG_DEVSTAT ) ) ); + return -ETIMEDOUT; +} + +/****************************************************************************** + * + * Admin queue + * + ****************************************************************************** + */ + +/** + * Get operating mode name (for debugging) + * + * @v mode Operating mode + * @ret name Mode name + */ +static inline const char * gve_mode_name ( unsigned int mode ) { + static char buf[ 8 /* "XXX-XXX" + NUL */ ]; + + snprintf ( buf, sizeof ( buf ), "%s-%s", + ( ( mode & GVE_MODE_DQO ) ? "DQO" : "GQI" ), + ( ( mode & GVE_MODE_QPL ) ? "QPL" : "RDA" ) ); + return buf; +} + +/** + * Allocate admin queue + * + * @v gve GVE device + * @ret rc Return status code + */ +static int gve_admin_alloc ( struct gve_nic *gve ) { + struct dma_device *dma = gve->dma; + struct gve_admin *admin = &gve->admin; + struct gve_scratch *scratch = &gve->scratch; + size_t admin_len = ( GVE_ADMIN_COUNT * sizeof ( admin->cmd[0] ) ); + size_t scratch_len = sizeof ( *scratch->buf ); + int rc; + + /* Allocate admin queue */ + admin->cmd = dma_alloc ( dma, &admin->map, admin_len, GVE_ALIGN ); + if ( ! admin->cmd ) { + rc = -ENOMEM; + goto err_admin; + } + + /* Allocate scratch buffer */ + scratch->buf = dma_alloc ( dma, &scratch->map, scratch_len, GVE_ALIGN ); + if ( ! scratch->buf ) { + rc = -ENOMEM; + goto err_scratch; + } + + DBGC ( gve, "GVE %p AQ at [%08lx,%08lx) scratch [%08lx,%08lx)\n", + gve, virt_to_phys ( admin->cmd ), + ( virt_to_phys ( admin->cmd ) + admin_len ), + virt_to_phys ( scratch->buf ), + ( virt_to_phys ( scratch->buf ) + scratch_len ) ); + return 0; + + dma_free ( &scratch->map, scratch->buf, scratch_len ); + err_scratch: + dma_free ( &admin->map, admin->cmd, admin_len ); + err_admin: + return rc; +} + +/** + * Free admin queue + * + * @v gve GVE device + */ +static void gve_admin_free ( struct gve_nic *gve ) { + struct gve_admin *admin = &gve->admin; + struct gve_scratch *scratch = &gve->scratch; + size_t admin_len = ( GVE_ADMIN_COUNT * sizeof ( admin->cmd[0] ) ); + size_t scratch_len = sizeof ( *scratch->buf ); + + /* Free scratch buffer */ + dma_free ( &scratch->map, scratch->buf, scratch_len ); + + /* Free admin queue */ + dma_free ( &admin->map, admin->cmd, admin_len ); +} + +/** + * Enable admin queue + * + * @v gve GVE device + */ +static void gve_admin_enable ( struct gve_nic *gve ) { + struct gve_admin *admin = &gve->admin; + size_t admin_len = ( GVE_ADMIN_COUNT * sizeof ( admin->cmd[0] ) ); + physaddr_t base; + + /* Reset queue */ + admin->prod = 0; + + /* Program queue addresses and capabilities */ + base = dma ( &admin->map, admin->cmd ); + writel ( bswap_32 ( base / GVE_PAGE_SIZE ), + gve->cfg + GVE_CFG_ADMIN_PFN ); + writel ( bswap_32 ( base & 0xffffffffUL ), + gve->cfg + GVE_CFG_ADMIN_BASE_LO ); + if ( sizeof ( base ) > sizeof ( uint32_t ) ) { + writel ( bswap_32 ( ( ( uint64_t ) base ) >> 32 ), + gve->cfg + GVE_CFG_ADMIN_BASE_HI ); + } else { + writel ( 0, gve->cfg + GVE_CFG_ADMIN_BASE_HI ); + } + writel ( bswap_16 ( admin_len ), gve->cfg + GVE_CFG_ADMIN_LEN ); + writel ( bswap_32 ( GVE_CFG_DRVSTAT_RUN ), gve->cfg + GVE_CFG_DRVSTAT ); +} + +/** + * Get next available admin queue command slot + * + * @v gve GVE device + * @ret cmd Admin queue command + */ +static union gve_admin_command * gve_admin_command ( struct gve_nic *gve ) { + struct gve_admin *admin = &gve->admin; + union gve_admin_command *cmd; + unsigned int index; + + /* Get next command slot */ + index = admin->prod; + cmd = &admin->cmd[ index % GVE_ADMIN_COUNT ]; + + /* Initialise request */ + memset ( cmd, 0, sizeof ( *cmd ) ); + + return cmd; +} + +/** + * Wait for admin queue command to complete + * + * @v gve GVE device + * @ret rc Return status code + */ +static int gve_admin_wait ( struct gve_nic *gve ) { + struct gve_admin *admin = &gve->admin; + uint32_t evt; + uint32_t pfn; + unsigned int i; + + /* Wait for any outstanding commands to complete */ + for ( i = 0 ; i < GVE_ADMIN_MAX_WAIT_MS ; i++ ) { + + /* Check event counter */ + rmb(); + evt = bswap_32 ( readl ( gve->cfg + GVE_CFG_ADMIN_EVT ) ); + if ( evt == admin->prod ) + return 0; + + /* Check for device reset */ + pfn = readl ( gve->cfg + GVE_CFG_ADMIN_PFN ); + if ( ! pfn ) + break; + + /* Delay */ + mdelay ( 1 ); + } + + DBGC ( gve, "GVE %p AQ %#02x %s (completed %#02x, status %#08x)\n", + gve, admin->prod, ( pfn ? "timed out" : "saw reset" ), evt, + bswap_32 ( readl ( gve->cfg + GVE_CFG_DEVSTAT ) ) ); + return ( pfn ? -ETIMEDOUT : -ECONNRESET ); +} + +/** + * Issue admin queue command + * + * @v gve GVE device + * @ret rc Return status code + */ +static int gve_admin ( struct gve_nic *gve ) { + struct gve_admin *admin = &gve->admin; + union gve_admin_command *cmd; + unsigned int index; + uint32_t opcode; + uint32_t status; + int rc; + + /* Ensure admin queue is idle */ + if ( ( rc = gve_admin_wait ( gve ) ) != 0 ) + return rc; + + /* Get next command slot */ + index = admin->prod; + cmd = &admin->cmd[ index % GVE_ADMIN_COUNT ]; + opcode = cmd->hdr.opcode; + DBGC2 ( gve, "GVE %p AQ %#02x command %#04x request:\n", + gve, index, opcode ); + DBGC2_HDA ( gve, 0, cmd, sizeof ( *cmd ) ); + + /* Increment producer counter */ + admin->prod++; + + /* Ring doorbell */ + wmb(); + writel ( bswap_32 ( admin->prod ), gve->cfg + GVE_CFG_ADMIN_DB ); + + /* Wait for command to complete */ + if ( ( rc = gve_admin_wait ( gve ) ) != 0 ) + return rc; + + /* Check command status */ + status = be32_to_cpu ( cmd->hdr.status ); + if ( status != GVE_ADMIN_STATUS_OK ) { + rc = -EIO_ADMIN ( status ); + DBGC ( gve, "GVE %p AQ %#02x command %#04x failed: %#08x\n", + gve, index, opcode, status ); + DBGC_HDA ( gve, 0, cmd, sizeof ( *cmd ) ); + DBGC ( gve, "GVE %p AQ error: %s\n", gve, strerror ( rc ) ); + return rc; + } + + DBGC2 ( gve, "GVE %p AQ %#02x command %#04x result:\n", + gve, index, opcode ); + DBGC2_HDA ( gve, 0, cmd, sizeof ( *cmd ) ); + return 0; +} + +/** + * Issue simple admin queue command + * + * @v gve GVE device + * @v opcode Operation code + * @v id ID parameter (or zero if not applicable) + * @ret rc Return status code + * + * Several admin queue commands take either an empty parameter list or + * a single 32-bit ID parameter. + */ +static int gve_admin_simple ( struct gve_nic *gve, unsigned int opcode, + unsigned int id ) { + union gve_admin_command *cmd; + int rc; + + /* Construct request */ + cmd = gve_admin_command ( gve ); + cmd->hdr.opcode = opcode; + cmd->simple.id = cpu_to_be32 ( id ); + + /* Issue command */ + if ( ( rc = gve_admin ( gve ) ) != 0 ) + return rc; + + return 0; +} + +/** + * Get device descriptor + * + * @v gve GVE device + * @ret rc Return status code + */ +static int gve_describe ( struct gve_nic *gve ) { + struct net_device *netdev = gve->netdev; + struct gve_device_descriptor *desc = &gve->scratch.buf->desc; + union gve_admin_command *cmd; + struct gve_option *opt; + unsigned int count; + unsigned int id; + size_t offset; + size_t max; + size_t len; + int rc; + + /* Construct request */ + cmd = gve_admin_command ( gve ); + cmd->hdr.opcode = GVE_ADMIN_DESCRIBE; + cmd->desc.addr = cpu_to_be64 ( dma ( &gve->scratch.map, desc ) ); + cmd->desc.ver = cpu_to_be32 ( GVE_ADMIN_DESCRIBE_VER ); + cmd->desc.len = cpu_to_be32 ( sizeof ( *desc ) ); + + /* Issue command */ + if ( ( rc = gve_admin ( gve ) ) != 0 ) + return rc; + DBGC2 ( gve, "GVE %p device descriptor:\n", gve ); + DBGC2_HDA ( gve, 0, desc, sizeof ( *desc ) ); + + /* Extract queue parameters */ + gve->events.count = be16_to_cpu ( desc->counters ); + gve->tx.count = be16_to_cpu ( desc->tx_count ); + gve->rx.count = be16_to_cpu ( desc->rx_count ); + DBGC ( gve, "GVE %p using %d TX, %d RX, %d events\n", + gve, gve->tx.count, gve->rx.count, gve->events.count ); + + /* Extract network parameters */ + build_assert ( sizeof ( desc->mac ) == ETH_ALEN ); + memcpy ( netdev->hw_addr, &desc->mac, sizeof ( desc->mac ) ); + netdev->mtu = be16_to_cpu ( desc->mtu ); + netdev->max_pkt_len = ( netdev->mtu + ETH_HLEN ); + DBGC ( gve, "GVE %p MAC %s (\"%s\") MTU %zd\n", + gve, eth_ntoa ( netdev->hw_addr ), + inet_ntoa ( desc->mac.in ), netdev->mtu ); + + /* Parse options */ + count = be16_to_cpu ( desc->opt_count ); + max = be16_to_cpu ( desc->len ); + gve->options = 0; + for ( offset = offsetof ( typeof ( *desc ), opts ) ; count ; + count--, offset += len ) { + + /* Check space for option header */ + if ( ( offset + sizeof ( *opt ) ) > max ) { + DBGC ( gve, "GVE %p underlength option at +%#02zx:\n", + gve, offset ); + DBGC_HDA ( gve, 0, desc, sizeof ( *desc ) ); + return -EINVAL; + } + opt = ( ( ( void * ) desc ) + offset ); + + /* Check space for option body */ + len = ( sizeof ( *opt ) + be16_to_cpu ( opt->len ) ); + if ( ( offset + len ) > max ) { + DBGC ( gve, "GVE %p malformed option at +%#02zx:\n", + gve, offset ); + DBGC_HDA ( gve, 0, desc, sizeof ( *desc ) ); + return -EINVAL; + } + + /* Record option as supported */ + id = be16_to_cpu ( opt->id ); + if ( id < ( 8 * sizeof ( gve->options ) ) ) + gve->options |= ( 1 << id ); + } + DBGC ( gve, "GVE %p supports options %#08x\n", gve, gve->options ); + + /* Select preferred operating mode */ + if ( gve->options & ( 1 << GVE_OPT_GQI_QPL ) ) { + /* GQI-QPL: in-order queues, queue page list addressing */ + gve->mode = GVE_MODE_QPL; + } else if ( gve->options & ( 1 << GVE_OPT_GQI_RDA ) ) { + /* GQI-RDA: in-order queues, raw DMA addressing */ + gve->mode = 0; + } else if ( gve->options & ( 1 << GVE_OPT_DQO_QPL ) ) { + /* DQO-QPL: out-of-order queues, queue page list addressing */ + gve->mode = ( GVE_MODE_DQO | GVE_MODE_QPL ); + } else if ( gve->options & ( 1 << GVE_OPT_DQO_RDA ) ) { + /* DQO-RDA: out-of-order queues, raw DMA addressing */ + gve->mode = GVE_MODE_DQO; + } else { + /* No options matched: assume the original GQI-QPL mode */ + gve->mode = GVE_MODE_QPL; + } + DBGC ( gve, "GVE %p using %s mode\n", + gve, gve_mode_name ( gve->mode ) ); + + return 0; +} + +/** + * Configure device resources + * + * @v gve GVE device + * @ret rc Return status code + */ +static int gve_configure ( struct gve_nic *gve ) { + struct gve_events *events = &gve->events; + struct gve_irqs *irqs = &gve->irqs; + union gve_admin_command *cmd; + uint32_t doorbell; + unsigned int db_off; + unsigned int i; + int rc; + + /* Construct request */ + cmd = gve_admin_command ( gve ); + cmd->hdr.opcode = GVE_ADMIN_CONFIGURE; + cmd->conf.events = + cpu_to_be64 ( dma ( &events->map, events->event ) ); + cmd->conf.irqs = + cpu_to_be64 ( dma ( &irqs->map, irqs->irq ) ); + cmd->conf.num_events = cpu_to_be32 ( events->count ); + cmd->conf.num_irqs = cpu_to_be32 ( GVE_IRQ_COUNT ); + cmd->conf.irq_stride = cpu_to_be32 ( sizeof ( irqs->irq[0] ) ); + cmd->conf.format = GVE_FORMAT ( gve->mode ); + + /* Issue command */ + if ( ( rc = gve_admin ( gve ) ) != 0 ) + return rc; + + /* Disable all interrupts */ + doorbell = ( ( gve->mode & GVE_MODE_DQO ) ? + 0 : bswap_32 ( GVE_GQI_IRQ_DISABLE ) ); + for ( i = 0 ; i < GVE_IRQ_COUNT ; i++ ) { + db_off = ( be32_to_cpu ( irqs->irq[i].db_idx ) * + sizeof ( uint32_t ) ); + DBGC ( gve, "GVE %p IRQ %d doorbell +%#04x\n", gve, i, db_off ); + irqs->db[i] = ( gve->db + db_off ); + writel ( doorbell, irqs->db[i] ); + } + + return 0; +} + +/** + * Deconfigure device resources + * + * @v gve GVE device + * @ret rc Return status code + */ +static int gve_deconfigure ( struct gve_nic *gve ) { + int rc; + + /* Issue command (with meaningless ID) */ + if ( ( rc = gve_admin_simple ( gve, GVE_ADMIN_DECONFIGURE, 0 ) ) != 0 ) + return rc; + + return 0; +} + +/** + * Register queue page list + * + * @v gve GVE device + * @v qpl Queue page list + * @ret rc Return status code + */ +static int gve_register ( struct gve_nic *gve, struct gve_qpl *qpl ) { + struct gve_pages *pages = &gve->scratch.buf->pages; + union gve_admin_command *cmd; + void *addr; + unsigned int i; + int rc; + + /* Do nothing if using raw DMA addressing */ + if ( ! ( gve->mode & GVE_MODE_QPL ) ) + return 0; + + /* Build page address list */ + for ( i = 0 ; i < qpl->count ; i++ ) { + addr = ( qpl->data + ( i * GVE_PAGE_SIZE ) ); + pages->addr[i] = cpu_to_be64 ( dma ( &qpl->map, addr ) ); + } + + /* Construct request */ + cmd = gve_admin_command ( gve ); + cmd->hdr.opcode = GVE_ADMIN_REGISTER; + cmd->reg.id = cpu_to_be32 ( qpl->id ); + cmd->reg.count = cpu_to_be32 ( qpl->count ); + cmd->reg.addr = cpu_to_be64 ( dma ( &gve->scratch.map, pages ) ); + cmd->reg.size = cpu_to_be64 ( GVE_PAGE_SIZE ); + + /* Issue command */ + if ( ( rc = gve_admin ( gve ) ) != 0 ) + return rc; + + return 0; +} + +/** + * Unregister page list + * + * @v gve GVE device + * @v qpl Queue page list + * @ret rc Return status code + */ +static int gve_unregister ( struct gve_nic *gve, struct gve_qpl *qpl ) { + int rc; + + /* Do nothing if using raw DMA addressing */ + if ( ! ( gve->mode & GVE_MODE_QPL ) ) + return 0; + + /* Issue command */ + if ( ( rc = gve_admin_simple ( gve, GVE_ADMIN_UNREGISTER, + qpl->id ) ) != 0 ) { + return rc; + } + + return 0; +} + +/** + * Construct command to create transmit queue + * + * @v queue Transmit queue + * @v qpl Queue page list ID + * @v cmd Admin queue command + */ +static void gve_create_tx_param ( struct gve_queue *queue, uint32_t qpl, + union gve_admin_command *cmd ) { + struct gve_admin_create_tx *create = &cmd->create_tx; + const struct gve_queue_type *type = queue->type; + + /* Construct request parameters */ + create->res = cpu_to_be64 ( dma ( &queue->res_map, queue->res ) ); + create->desc = + cpu_to_be64 ( dma ( &queue->desc_map, queue->desc.raw ) ); + create->qpl_id = cpu_to_be32 ( qpl ); + create->notify_id = cpu_to_be32 ( type->irq ); + create->desc_count = cpu_to_be16 ( queue->count ); + if ( queue->cmplt.raw ) { + create->cmplt = cpu_to_be64 ( dma ( &queue->cmplt_map, + queue->cmplt.raw ) ); + create->cmplt_count = cpu_to_be16 ( queue->count ); + } +} + +/** + * Construct command to create receive queue + * + * @v queue Receive queue + * @v qpl Queue page list ID + * @v cmd Admin queue command + */ +static void gve_create_rx_param ( struct gve_queue *queue, uint32_t qpl, + union gve_admin_command *cmd ) { + struct gve_admin_create_rx *create = &cmd->create_rx; + const struct gve_queue_type *type = queue->type; + + /* Construct request parameters */ + create->notify_id = cpu_to_be32 ( type->irq ); + create->res = cpu_to_be64 ( dma ( &queue->res_map, queue->res ) ); + create->desc = + cpu_to_be64 ( dma ( &queue->desc_map, queue->desc.raw ) ); + create->cmplt = + cpu_to_be64 ( dma ( &queue->cmplt_map, queue->cmplt.raw ) ); + create->qpl_id = cpu_to_be32 ( qpl ); + create->desc_count = cpu_to_be16 ( queue->count ); + create->bufsz = cpu_to_be16 ( GVE_BUF_SIZE ); + create->cmplt_count = cpu_to_be16 ( queue->count ); +} + +/** + * Create transmit or receive queue + * + * @v gve GVE device + * @v queue Descriptor queue + * @ret rc Return status code + */ +static int gve_create_queue ( struct gve_nic *gve, struct gve_queue *queue ) { + const struct gve_queue_type *type = queue->type; + const struct gve_queue_stride *stride = &queue->stride; + union gve_admin_command *cmd; + struct gve_buffer *buf; + unsigned int db_off; + unsigned int evt_idx; + unsigned int tag; + unsigned int i; + uint32_t qpl; + int rc; + + /* Reset queue */ + queue->prod = 0; + queue->cons = 0; + queue->done = 0; + memset ( queue->desc.raw, 0, ( queue->count * stride->desc ) ); + memset ( queue->cmplt.raw, 0, ( queue->count * stride->cmplt ) ); + for ( i = 0 ; i < queue->fill ; i++ ) + queue->tag[i] = i; + + /* Pre-populate descriptor offsets for in-order queues */ + if ( ! ( gve->mode & GVE_MODE_DQO ) ) { + buf = ( queue->desc.raw + stride->desc - sizeof ( *buf ) ); + for ( i = 0 ; i < queue->count ; i++ ) { + tag = ( i & ( queue->fill - 1 ) ); + buf->addr = cpu_to_be64 ( gve_address ( queue, tag ) ); + buf = ( ( ( void * ) buf ) + stride->desc ); + } + } + + /* Construct request */ + cmd = gve_admin_command ( gve ); + cmd->hdr.opcode = type->create; + qpl = ( ( gve->mode & GVE_MODE_QPL ) ? type->qpl : GVE_RAW_QPL ); + type->param ( queue, qpl, cmd ); + + /* Issue command */ + if ( ( rc = gve_admin ( gve ) ) != 0 ) + return rc; + + /* Record indices */ + db_off = ( be32_to_cpu ( queue->res->db_idx ) * sizeof ( uint32_t ) ); + evt_idx = be32_to_cpu ( queue->res->evt_idx ); + DBGC ( gve, "GVE %p %s doorbell +%#04x event counter %d\n", + gve, type->name, db_off, evt_idx ); + queue->db = ( gve->db + db_off ); + assert ( evt_idx < gve->events.count ); + queue->event = &gve->events.event[evt_idx]; + assert ( queue->event->count == 0 ); + + /* Unmask dummy interrupt */ + pci_msix_unmask ( &gve->msix, type->irq ); + + /* Rearm queue interrupt if applicable */ + if ( gve->mode & GVE_MODE_DQO ) + writel ( GVE_DQO_IRQ_REARM, gve->irqs.db[type->irq] ); + + return 0; +} + +/** + * Destroy transmit or receive queue + * + * @v gve GVE device + * @v queue Descriptor queue + * @ret rc Return status code + */ +static int gve_destroy_queue ( struct gve_nic *gve, struct gve_queue *queue ) { + const struct gve_queue_type *type = queue->type; + int rc; + + /* Mask dummy interrupt */ + pci_msix_mask ( &gve->msix, type->irq ); + + /* Issue command */ + if ( ( rc = gve_admin_simple ( gve, type->destroy, 0 ) ) != 0 ) + return rc; + + return 0; +} + +/****************************************************************************** + * + * Network device interface + * + ****************************************************************************** + */ + +/** + * Allocate shared queue resources + * + * @v gve GVE device + * @ret rc Return status code + */ +static int gve_alloc_shared ( struct gve_nic *gve ) { + struct dma_device *dma = gve->dma; + struct gve_irqs *irqs = &gve->irqs; + struct gve_events *events = &gve->events; + size_t irqs_len = ( GVE_IRQ_COUNT * sizeof ( irqs->irq[0] ) ); + size_t events_len = ( gve->events.count * sizeof ( events->event[0] ) ); + int rc; + + /* Allocate interrupt channels */ + irqs->irq = dma_alloc ( dma, &irqs->map, irqs_len, GVE_ALIGN ); + if ( ! irqs->irq ) { + rc = -ENOMEM; + goto err_irqs; + } + DBGC ( gve, "GVE %p IRQs at [%08lx,%08lx)\n", + gve, virt_to_phys ( irqs->irq ), + ( virt_to_phys ( irqs->irq ) + irqs_len ) ); + + /* Allocate event counters */ + events->event = dma_alloc ( dma, &events->map, events_len, GVE_ALIGN ); + if ( ! events->event ) { + rc = -ENOMEM; + goto err_events; + } + DBGC ( gve, "GVE %p events at [%08lx,%08lx)\n", + gve, virt_to_phys ( events->event ), + ( virt_to_phys ( events->event ) + events_len ) ); + + return 0; + + dma_free ( &events->map, events->event, events_len ); + err_events: + dma_free ( &irqs->map, irqs->irq, irqs_len ); + err_irqs: + return rc; +} + +/** + * Free shared queue resources + * + * @v gve GVE device + */ +static void gve_free_shared ( struct gve_nic *gve ) { + struct gve_irqs *irqs = &gve->irqs; + struct gve_events *events = &gve->events; + size_t irqs_len = ( GVE_IRQ_COUNT * sizeof ( irqs->irq[0] ) ); + size_t events_len = ( gve->events.count * sizeof ( events->event[0] ) ); + + /* Free event counters */ + dma_free ( &events->map, events->event, events_len ); + + /* Free interrupt channels */ + dma_free ( &irqs->map, irqs->irq, irqs_len ); +} + +/** + * Allocate queue page list + * + * @v gve GVE device + * @v qpl Queue page list + * @v id Queue page list ID + * @v buffers Number of data buffers + * @ret rc Return status code + */ +static int gve_alloc_qpl ( struct gve_nic *gve, struct gve_qpl *qpl, + uint32_t id, unsigned int buffers ) { + size_t len; + + /* Record ID */ + qpl->id = id; + + /* Calculate number of pages required */ + build_assert ( GVE_BUF_SIZE <= GVE_PAGE_SIZE ); + qpl->count = ( ( buffers + GVE_BUF_PER_PAGE - 1 ) / GVE_BUF_PER_PAGE ); + assert ( qpl->count <= GVE_QPL_MAX ); + + /* Allocate pages (as a single block) */ + len = ( qpl->count * GVE_PAGE_SIZE ); + qpl->data = dma_umalloc ( gve->dma, &qpl->map, len, GVE_ALIGN ); + if ( ! qpl->data ) + return -ENOMEM; + qpl->base = ( ( gve->mode == GVE_MODE_QPL ) ? + 0 : dma ( &qpl->map, qpl->data ) ); + + DBGC ( gve, "GVE %p QPL %#08x at [%08lx,%08lx)\n", + gve, qpl->id, virt_to_phys ( qpl->data ), + ( virt_to_phys ( qpl->data ) + len ) ); + return 0; +} + +/** + * Free queue page list + * + * @v gve GVE device + * @v qpl Queue page list + */ +static void gve_free_qpl ( struct gve_nic *nic __unused, + struct gve_qpl *qpl ) { + size_t len = ( qpl->count * GVE_PAGE_SIZE ); + + /* Free pages */ + dma_ufree ( &qpl->map, qpl->data, len ); +} + +/** + * Calculate next receive sequence number + * + * @v seq Current sequence number, or zero to start sequence + * @ret next Next sequence number + */ +static inline __attribute__ (( always_inline )) unsigned int +gve_next ( unsigned int seq ) { + + /* The receive completion sequence number is a modulo 7 + * counter that cycles through the non-zero three-bit values 1 + * to 7 inclusive. + * + * Since 7 is coprime to 2^n, this ensures that the sequence + * number changes each time that a new completion is written + * to memory. + * + * Since the counter takes only non-zero values, this ensures + * that the sequence number changes whenever a new completion + * is first written to a zero-initialised completion ring. + */ + seq = ( ( seq + 1 ) & GVE_GQI_RX_SEQ_MASK ); + return ( seq ? seq : 1 ); +} + +/** + * Allocate descriptor queue + * + * @v gve GVE device + * @v queue Descriptor queue + * @ret rc Return status code + */ +static int gve_alloc_queue ( struct gve_nic *gve, struct gve_queue *queue ) { + const struct gve_queue_type *type = queue->type; + struct gve_queue_stride *stride = &queue->stride; + struct dma_device *dma = gve->dma; + size_t desc_len; + size_t cmplt_len; + size_t res_len; + int rc; + + /* Sanity checks */ + if ( ( queue->count == 0 ) || + ( queue->count & ( queue->count - 1 ) ) ) { + DBGC ( gve, "GVE %p %s invalid queue size %d\n", + gve, type->name, queue->count ); + rc = -EINVAL; + goto err_sanity; + } + + /* Set queue strides and calculate total lengths */ + *stride = ( ( gve->mode & GVE_MODE_DQO ) ? + type->stride.dqo : type->stride.gqi ); + desc_len = ( queue->count * stride->desc ); + cmplt_len = ( queue->count * stride->cmplt ); + res_len = sizeof ( *queue->res ); + + /* Calculate maximum fill level */ + assert ( ( type->fill & ( type->fill - 1 ) ) == 0 ); + queue->fill = type->fill; + if ( queue->fill > queue->count ) + queue->fill = queue->count; + DBGC ( gve, "GVE %p %s using QPL %#08x with %d/%d descriptors\n", + gve, type->name, type->qpl, queue->fill, queue->count ); + + /* Allocate queue page list */ + if ( ( rc = gve_alloc_qpl ( gve, &queue->qpl, type->qpl, + queue->fill ) ) != 0 ) + goto err_qpl; + + /* Allocate descriptors */ + queue->desc.raw = dma_umalloc ( dma, &queue->desc_map, desc_len, + GVE_ALIGN ); + if ( ! queue->desc.raw ) { + rc = -ENOMEM; + goto err_desc; + } + DBGC ( gve, "GVE %p %s descriptors at [%08lx,%08lx)\n", + gve, type->name, virt_to_phys ( queue->desc.raw ), + ( virt_to_phys ( queue->desc.raw ) + desc_len ) ); + + /* Allocate completions */ + if ( cmplt_len ) { + queue->cmplt.raw = dma_umalloc ( dma, &queue->cmplt_map, + cmplt_len, GVE_ALIGN ); + if ( ! queue->cmplt.raw ) { + rc = -ENOMEM; + goto err_cmplt; + } + DBGC ( gve, "GVE %p %s completions at [%08lx,%08lx)\n", + gve, type->name, virt_to_phys ( queue->cmplt.raw ), + ( virt_to_phys ( queue->cmplt.raw ) + cmplt_len ) ); + } + + /* Allocate queue resources */ + queue->res = dma_alloc ( dma, &queue->res_map, res_len, GVE_ALIGN ); + if ( ! queue->res ) { + rc = -ENOMEM; + goto err_res; + } + memset ( queue->res, 0, res_len ); + + return 0; + + dma_free ( &queue->res_map, queue->res, res_len ); + err_res: + if ( cmplt_len ) + dma_ufree ( &queue->cmplt_map, queue->cmplt.raw, cmplt_len ); + err_cmplt: + dma_ufree ( &queue->desc_map, queue->desc.raw, desc_len ); + err_desc: + gve_free_qpl ( gve, &queue->qpl ); + err_qpl: + err_sanity: + return rc; +} + +/** + * Free descriptor queue + * + * @v gve GVE device + * @v queue Descriptor queue + */ +static void gve_free_queue ( struct gve_nic *gve, struct gve_queue *queue ) { + const struct gve_queue_stride *stride = &queue->stride; + size_t desc_len = ( queue->count * stride->desc ); + size_t cmplt_len = ( queue->count * stride->cmplt ); + size_t res_len = sizeof ( *queue->res ); + + /* Free queue resources */ + dma_free ( &queue->res_map, queue->res, res_len ); + + /* Free completions, if applicable */ + if ( cmplt_len ) + dma_ufree ( &queue->cmplt_map, queue->cmplt.raw, cmplt_len ); + + /* Free descriptors */ + dma_ufree ( &queue->desc_map, queue->desc.raw, desc_len ); + + /* Free queue page list */ + gve_free_qpl ( gve, &queue->qpl ); +} + +/** + * Cancel any pending transmissions + * + * @v gve GVE device + */ +static void gve_cancel_tx ( struct gve_nic *gve ) { + struct net_device *netdev = gve->netdev; + struct io_buffer *iobuf; + unsigned int i; + + /* Cancel any pending transmissions */ + for ( i = 0 ; i < ( sizeof ( gve->tx_iobuf ) / + sizeof ( gve->tx_iobuf[0] ) ) ; i++ ) { + iobuf = gve->tx_iobuf[i]; + gve->tx_iobuf[i] = NULL; + if ( iobuf ) + netdev_tx_complete_err ( netdev, iobuf, -ECANCELED ); + } +} + +/** + * Start up device + * + * @v gve GVE device + * @ret rc Return status code + */ +static int gve_start ( struct gve_nic *gve ) { + struct gve_queue *tx = &gve->tx; + struct gve_queue *rx = &gve->rx; + int rc; + + /* Cancel any pending transmissions */ + gve_cancel_tx ( gve ); + + /* Reset receive sequence */ + gve->seq = gve_next ( 0 ); + + /* Configure device resources */ + if ( ( rc = gve_configure ( gve ) ) != 0 ) + goto err_configure; + + /* Register transmit queue page list */ + if ( ( rc = gve_register ( gve, &tx->qpl ) ) != 0 ) + goto err_register_tx; + + /* Register receive queue page list */ + if ( ( rc = gve_register ( gve, &rx->qpl ) ) != 0 ) + goto err_register_rx; + + /* Create transmit queue */ + if ( ( rc = gve_create_queue ( gve, tx ) ) != 0 ) + goto err_create_tx; + + /* Create receive queue */ + if ( ( rc = gve_create_queue ( gve, rx ) ) != 0 ) + goto err_create_rx; + + return 0; + + gve_destroy_queue ( gve, rx ); + err_create_rx: + gve_destroy_queue ( gve, tx ); + err_create_tx: + gve_unregister ( gve, &rx->qpl ); + err_register_rx: + gve_unregister ( gve, &tx->qpl ); + err_register_tx: + gve_deconfigure ( gve ); + err_configure: + return rc; +} + +/** + * Stop device + * + * @v gve GVE device + */ +static void gve_stop ( struct gve_nic *gve ) { + struct gve_queue *tx = &gve->tx; + struct gve_queue *rx = &gve->rx; + + /* Destroy queues */ + gve_destroy_queue ( gve, rx ); + gve_destroy_queue ( gve, tx ); + + /* Unregister page lists */ + gve_unregister ( gve, &rx->qpl ); + gve_unregister ( gve, &tx->qpl ); + + /* Deconfigure device */ + gve_deconfigure ( gve ); +} + +/** + * Device startup process + * + * @v gve GVE device + */ +static void gve_startup ( struct gve_nic *gve ) { + struct net_device *netdev = gve->netdev; + int rc; + + /* Reset device */ + if ( ( rc = gve_reset ( gve ) ) != 0 ) + goto err_reset; + + /* Enable admin queue */ + gve_admin_enable ( gve ); + + /* Start device */ + if ( ( rc = gve_start ( gve ) ) != 0 ) + goto err_start; + + /* Reset retry count */ + gve->retries = 0; + + /* (Ab)use link status to report startup status */ + netdev_link_up ( netdev ); + + return; + + gve_stop ( gve ); + err_start: + err_reset: + DBGC ( gve, "GVE %p startup failed: %s\n", gve, strerror ( rc ) ); + netdev_link_err ( netdev, rc ); + if ( gve->retries++ < GVE_RESET_MAX_RETRY ) + process_add ( &gve->startup ); +} + +/** + * Trigger startup process + * + * @v gve GVE device + */ +static void gve_restart ( struct gve_nic *gve ) { + struct net_device *netdev = gve->netdev; + + /* Mark link down to inhibit polling and transmit activity */ + netdev_link_down ( netdev ); + + /* Schedule startup process */ + process_add ( &gve->startup ); +} + +/** + * Reset recovery watchdog + * + * @v timer Reset recovery watchdog timer + * @v over Failure indicator + */ +static void gve_watchdog ( struct retry_timer *timer, int over __unused ) { + struct gve_nic *gve = container_of ( timer, struct gve_nic, watchdog ); + uint32_t activity; + uint32_t pfn; + int rc; + + /* Reschedule watchdog */ + start_timer_fixed ( &gve->watchdog, GVE_WATCHDOG_TIMEOUT ); + + /* Reset device (for test purposes) if applicable */ + if ( ( rc = inject_fault ( VM_MIGRATED_RATE ) ) != 0 ) { + DBGC ( gve, "GVE %p synthesising host reset\n", gve ); + writel ( 0, gve->cfg + GVE_CFG_ADMIN_PFN ); + } + + /* Check for activity since last timer invocation */ + activity = ( gve->tx.cons + gve->rx.cons ); + if ( activity != gve->activity ) { + gve->activity = activity; + return; + } + + /* Check for reset */ + pfn = readl ( gve->cfg + GVE_CFG_ADMIN_PFN ); + if ( pfn ) { + DBGC2 ( gve, "GVE %p idle but not in reset\n", gve ); + return; + } + + /* Schedule restart */ + DBGC ( gve, "GVE %p watchdog detected reset by host\n", gve ); + gve_restart ( gve ); +} + +/** + * Open network device + * + * @v netdev Network device + * @ret rc Return status code + */ +static int gve_open ( struct net_device *netdev ) { + struct gve_nic *gve = netdev->priv; + struct gve_queue *tx = &gve->tx; + struct gve_queue *rx = &gve->rx; + int rc; + + /* Allocate shared queue resources */ + if ( ( rc = gve_alloc_shared ( gve ) ) != 0 ) + goto err_alloc_shared; + + /* Allocate and prepopulate transmit queue */ + if ( ( rc = gve_alloc_queue ( gve, tx ) ) != 0 ) + goto err_alloc_tx; + + /* Allocate and prepopulate receive queue */ + if ( ( rc = gve_alloc_queue ( gve, rx ) ) != 0 ) + goto err_alloc_rx; + + /* Trigger startup */ + gve_restart ( gve ); + + /* Start reset recovery watchdog timer */ + start_timer_fixed ( &gve->watchdog, GVE_WATCHDOG_TIMEOUT ); + + return 0; + + gve_free_queue ( gve, rx ); + err_alloc_rx: + gve_free_queue ( gve, tx ); + err_alloc_tx: + gve_free_shared ( gve ); + err_alloc_shared: + return rc; +} + +/** + * Close network device + * + * @v netdev Network device + */ +static void gve_close ( struct net_device *netdev ) { + struct gve_nic *gve = netdev->priv; + struct gve_queue *tx = &gve->tx; + struct gve_queue *rx = &gve->rx; + + /* Stop reset recovery timer */ + stop_timer ( &gve->watchdog ); + + /* Terminate startup process */ + process_del ( &gve->startup ); + + /* Stop and reset device */ + gve_stop ( gve ); + gve_reset ( gve ); + + /* Cancel any pending transmissions */ + gve_cancel_tx ( gve ); + + /* Free queues */ + gve_free_queue ( gve, rx ); + gve_free_queue ( gve, tx ); + + /* Free shared queue resources */ + gve_free_shared ( gve ); +} + +/** + * Transmit packet + * + * @v netdev Network device + * @v iobuf I/O buffer + * @ret rc Return status code + */ +static int gve_transmit ( struct net_device *netdev, struct io_buffer *iobuf ) { + struct gve_nic *gve = netdev->priv; + struct gve_queue *tx = &gve->tx; + struct gve_gqi_tx_descriptor *gqi; + struct gve_dqo_tx_descriptor *dqo; + unsigned int count; + unsigned int index; + unsigned int tag; + unsigned int chain; + uint32_t doorbell; + size_t frag_len; + size_t offset; + size_t next; + size_t len; + + /* Do nothing if queues are not yet set up */ + if ( ! netdev_link_ok ( netdev ) ) + return -ENETDOWN; + + /* Defer packet if there is no space in the transmit ring */ + len = iob_len ( iobuf ); + count = ( ( len + GVE_BUF_SIZE - 1 ) / GVE_BUF_SIZE ); + if ( ( ( tx->prod - tx->cons ) + count ) > tx->fill ) { + netdev_tx_defer ( netdev, iobuf ); + return 0; + } + + /* Copy packet to queue pages and populate descriptors */ + for ( offset = 0, chain = 0 ; ; offset = next, chain = tag ) { + + /* Identify next available buffer */ + index = ( tx->prod++ & ( tx->count - 1 ) ); + tag = tx->tag[ index % GVE_TX_FILL ]; + + /* Sanity check */ + assert ( gve->tx_iobuf[tag] == NULL ); + + /* Copy packet fragment */ + frag_len = ( len - offset ); + if ( frag_len > GVE_BUF_SIZE ) + frag_len = GVE_BUF_SIZE; + memcpy ( gve_buffer ( tx, tag ), + ( iobuf->data + offset ), frag_len ); + next = ( offset + frag_len ); + + /* Populate descriptor */ + if ( gve->mode & GVE_MODE_DQO ) { + + /* Out-of-order descriptor */ + dqo = &tx->desc.tx.dqo[index]; + dqo->buf.addr = + cpu_to_le64 ( gve_address ( tx, tag ) ); + if ( next == len ) { + dqo->type = ( GVE_DQO_TX_TYPE_PACKET | + GVE_DQO_TX_TYPE_LAST ); + dqo->tag.id = tag; + dqo->tag.count = count; + } else { + dqo->type = GVE_DQO_TX_TYPE_PACKET; + dqo->tag.id = 0; + dqo->tag.count = 0; + } + dqo->len = cpu_to_le16 ( frag_len ); + gve->tx_chain[tag] = chain; + + } else { + + /* In-order descriptor */ + gqi = &tx->desc.tx.gqi[index]; + if ( offset ) { + gqi->type = GVE_GQI_TX_TYPE_CONT; + gqi->count = 0; + gqi->total = 0; + } else { + gqi->type = GVE_GQI_TX_TYPE_START; + gqi->count = count; + gqi->total = cpu_to_be16 ( len ); + } + gqi->len = cpu_to_be16 ( frag_len ); + + } + DBGC2 ( gve, "GVE %p TXD %#04x %#02x:%#02x len %#04zx/%#04zx " + "at %#08lx\n", gve, index, tag, count, frag_len, len, + gve_address ( tx, tag ) ); + + /* Record I/O buffer against final descriptor */ + if ( next == len ) { + gve->tx_iobuf[tag] = iobuf; + break; + } + } + assert ( ( tx->prod - tx->cons ) <= tx->fill ); + + /* Ring doorbell */ + doorbell = tx->prod; + if ( gve->mode & GVE_MODE_DQO ) { + doorbell &= ( tx->count - 1 ); + } else { + doorbell = bswap_32 ( doorbell ); + } + wmb(); + writel ( doorbell, tx->db ); + + return 0; +} + +/** + * Poll for completed transmissions + * + * @v netdev Network device + */ +static void gve_poll_tx ( struct net_device *netdev ) { + struct gve_nic *gve = netdev->priv; + struct gve_queue *tx = &gve->tx; + struct gve_dqo_tx_completion *dqo; + struct io_buffer *iobuf; + unsigned int index; + unsigned int gen; + unsigned int bit; + unsigned int tag; + uint32_t count; + + /* Process transmit completions */ + if ( gve->mode & GVE_MODE_DQO ) { + + /* Out-of-order completions */ + while ( 1 ) { + + /* Read next possible completion */ + gen = ( tx->done & tx->count ); + index = ( tx->done & ( tx->count - 1 ) ); + dqo = &tx->cmplt.tx.dqo[index]; + + /* Check generation bit */ + bit = ( dqo->flags & GVE_DQO_TXF_GEN ); + if ( ( !! bit ) == ( !! gen ) ) + break; + rmb(); + tx->done++; + + /* Ignore non-packet completions */ + if ( ( ! ( dqo->flags & GVE_DQO_TXF_PKT ) ) || + ( dqo->tag.count < 0 ) ) { + DBGC2 ( gve, "GVE %p TXC %#04x flags %#02x " + "ignored\n", gve, index, dqo->flags ); + continue; + } + + /* Parse completion */ + tag = dqo->tag.id; + count = dqo->tag.count; + iobuf = gve->tx_iobuf[tag]; + gve->tx_iobuf[tag] = NULL; + assert ( iobuf != NULL ); + + /* Return completed descriptors to ring */ + while ( count-- ) { + DBGC2 ( gve, "GVE %p TXC %#04x %#02x:%#02x " + "complete\n", gve, index, tag, + dqo->tag.count ); + tx->tag[ tx->cons++ % GVE_TX_FILL ] = tag; + tag = gve->tx_chain[tag]; + } + + /* Hand off to network stack */ + if ( iobuf ) + netdev_tx_complete ( netdev, iobuf ); + } + + } else { + + /* Read event counter */ + count = be32_to_cpu ( tx->event->count ); + + /* Process transmit completions */ + while ( count != tx->cons ) { + DBGC2 ( gve, "GVE %p TXC %#04x complete\n", + gve, tx->cons ); + tag = ( tx->cons % GVE_TX_FILL ); + iobuf = gve->tx_iobuf[tag]; + gve->tx_iobuf[tag] = NULL; + tx->cons++; + if ( iobuf ) + netdev_tx_complete ( netdev, iobuf ); + } + } +} + +/** + * Poll for received packets + * + * @v netdev Network device + */ +static void gve_poll_rx ( struct net_device *netdev ) { + struct gve_nic *gve = netdev->priv; + struct gve_queue *rx = &gve->rx; + struct gve_gqi_rx_completion *gqi; + struct gve_dqo_rx_completion *dqo; + struct io_buffer *iobuf; + unsigned int index; + unsigned int gen; + unsigned int bit; + unsigned int seq; + unsigned int tag; + uint32_t done; + size_t total; + size_t len; + int rc; + + /* Process receive completions */ + done = rx->done; + seq = gve->seq; + total = 0; + while ( 1 ) { + + /* Read next possible completion */ + rc = 0; + gen = ( done & rx->count ); + index = ( done++ & ( rx->count - 1 ) ); + if ( gve->mode & GVE_MODE_DQO ) { + + /* Out-of-order completion */ + dqo = &rx->cmplt.rx.dqo[index]; + + /* Check generation bit */ + bit = ( dqo->len & cpu_to_le16 ( GVE_DQO_RXL_GEN ) ); + if ( ( !! bit ) == ( !! gen ) ) + break; + rmb(); + + /* Parse completion */ + len = ( le16_to_cpu ( dqo->len ) & + ( GVE_BUF_SIZE - 1 ) ); + tag = dqo->tag; + DBGC2 ( gve, "GVE %p RXC %#04x %#02x:%#02x len %#04zx " + "at %#08zx\n", gve, index, tag, dqo->flags, + len, gve_offset ( rx, tag ) ); + + /* Accumulate a complete packet */ + if ( dqo->status & GVE_DQO_RXS_ERROR ) { + rc = -EIO; + total = 0; + } else { + total += len; + if ( ! ( dqo->flags & GVE_DQO_RXF_LAST ) ) + continue; + } + + } else { + + /* In-order completion */ + gqi = &rx->cmplt.rx.gqi[index]; + + /* Check sequence number */ + if ( ( gqi->seq & GVE_GQI_RX_SEQ_MASK ) != seq ) + break; + rmb(); + seq = gve_next ( seq ); + + /* Parse completion */ + len = be16_to_cpu ( gqi->len ); + tag = ( index % GVE_RX_FILL ); + DBGC2 ( gve, "GVE %p RXC %#04x %#02x:%#02x len %#04zx " + "at %#08zx\n", gve, index, gqi->seq, + gqi->flags, len, gve_offset ( rx, tag ) ); + + /* Accumulate a complete packet */ + if ( gqi->flags & GVE_GQI_RXF_ERROR ) { + rc = -EIO; + total = 0; + } else { + total += len; + if ( gqi->flags & GVE_GQI_RXF_MORE ) + continue; + } + gve->seq = seq; + } + + /* Allocate and populate I/O buffer */ + iobuf = ( total ? alloc_iob ( total ) : NULL ); + for ( ; rx->done != done ; rx->done++ ) { + + /* Re-read completion and return tag to ring */ + index = ( rx->done & ( rx->count - 1 ) ); + if ( gve->mode & GVE_MODE_DQO ) { + dqo = &rx->cmplt.rx.dqo[index]; + tag = dqo->tag; + len = ( le16_to_cpu ( dqo->len ) & + ( GVE_BUF_SIZE - 1 ) ); + rx->tag[ rx->cons++ % GVE_RX_FILL ] = tag; + } else { + gqi = &rx->cmplt.rx.gqi[index]; + tag = ( index % GVE_RX_FILL ); + len = be16_to_cpu ( gqi->len ); + assert ( rx->cons == rx->done ); + rx->cons++; + } + + /* Copy data */ + if ( iobuf ) { + memcpy ( iob_put ( iobuf, len ), + gve_buffer ( rx, tag ), len ); + } + } + assert ( ( iobuf == NULL ) || ( iob_len ( iobuf ) == total ) ); + total = 0; + + /* Hand off packet to network stack */ + if ( iobuf ) { + if ( ! ( gve->mode & GVE_MODE_DQO ) ) + iob_pull ( iobuf, GVE_GQI_RX_PAD ); + netdev_rx ( netdev, iobuf ); + } else { + netdev_rx_err ( netdev, NULL, ( rc ? rc : -ENOMEM ) ); + } + } +} + +/** + * Refill receive queue + * + * @v netdev Network device + */ +static void gve_refill_rx ( struct net_device *netdev ) { + struct gve_nic *gve = netdev->priv; + struct gve_queue *rx = &gve->rx; + struct gve_dqo_rx_descriptor *dqo; + unsigned int refill; + unsigned int index; + unsigned int tag; + uint32_t doorbell; + + /* Calculate refill quantity */ + doorbell = ( rx->cons + rx->fill ); + refill = ( doorbell - rx->prod ); + if ( ! refill ) + return; + + /* Refill ring */ + if ( gve->mode & GVE_MODE_DQO ) { + + /* Out-of-order descriptors */ + while ( refill-- ) { + + /* Identify next available buffer */ + index = ( rx->prod++ & ( rx->count - 1 ) ); + tag = rx->tag[ index % GVE_RX_FILL ]; + + /* Populate descriptor */ + dqo = &rx->desc.rx.dqo[index]; + dqo->tag = tag; + dqo->buf.addr = + cpu_to_le64 ( gve_address ( rx, tag ) ); + DBGC2 ( gve, "GVE %p RXD %#04x:%#02x at %#08llx\n", + gve, index, dqo->tag, + ( ( unsigned long long ) + le64_to_cpu ( dqo->buf.addr ) ) ); + } + wmb(); + assert ( rx->prod == doorbell ); + + } else { + + /* The in-order receive descriptors are prepopulated + * at the time of creating the receive queue (pointing + * to the preallocated queue pages). Refilling is + * therefore just a case of ringing the doorbell if + * the device is not yet aware of any available + * descriptors. + */ + rx->prod += refill; + assert ( rx->prod == doorbell ); + DBGC2 ( gve, "GVE %p RXD %#04x ready\n", gve, rx->prod ); + + /* Doorbell is big-endian */ + doorbell = bswap_32 ( doorbell ); + } + + /* Ring doorbell */ + writel ( doorbell, rx->db ); +} + +/** + * Poll for completed and received packets + * + * @v netdev Network device + */ +static void gve_poll ( struct net_device *netdev ) { + struct gve_nic *gve = netdev->priv; + + /* Do nothing if queues are not yet set up */ + if ( ! netdev_link_ok ( netdev ) ) + return; + + /* Poll for transmit completions */ + gve_poll_tx ( netdev ); + + /* Poll for receive completions */ + gve_poll_rx ( netdev ); + + /* Refill receive queue */ + gve_refill_rx ( netdev ); + + /* Rearm queue interrupts if applicable */ + if ( gve->mode & GVE_MODE_DQO ) { + writel ( GVE_DQO_IRQ_REARM, gve->irqs.db[GVE_TX_IRQ] ); + writel ( GVE_DQO_IRQ_REARM, gve->irqs.db[GVE_RX_IRQ] ); + } +} + +/** GVE network device operations */ +static struct net_device_operations gve_operations = { + .open = gve_open, + .close = gve_close, + .transmit = gve_transmit, + .poll = gve_poll, +}; + +/****************************************************************************** + * + * PCI interface + * + ****************************************************************************** + */ + +/** Transmit descriptor queue type */ +static const struct gve_queue_type gve_tx_type = { + .name = "TX", + .param = gve_create_tx_param, + .qpl = GVE_TX_QPL, + .irq = GVE_TX_IRQ, + .fill = GVE_TX_FILL, + .stride = { + .gqi = { + .desc = sizeof ( struct gve_gqi_tx_descriptor ), + }, + .dqo = { + .desc = sizeof ( struct gve_dqo_tx_descriptor ), + .cmplt = sizeof ( struct gve_dqo_tx_completion ), + }, + }, + .create = GVE_ADMIN_CREATE_TX, + .destroy = GVE_ADMIN_DESTROY_TX, +}; + +/** Receive descriptor queue type */ +static const struct gve_queue_type gve_rx_type = { + .name = "RX", + .param = gve_create_rx_param, + .qpl = GVE_RX_QPL, + .irq = GVE_RX_IRQ, + .fill = GVE_RX_FILL, + .stride = { + .gqi = { + .desc = sizeof ( struct gve_gqi_rx_descriptor ), + .cmplt = sizeof ( struct gve_gqi_rx_completion ), + }, + .dqo = { + .desc = sizeof ( struct gve_dqo_rx_descriptor ), + .cmplt = sizeof ( struct gve_dqo_rx_completion ), + }, + }, + .create = GVE_ADMIN_CREATE_RX, + .destroy = GVE_ADMIN_DESTROY_RX, +}; + +/** + * Set up admin queue and get device description + * + * @v gve GVE device + * @ret rc Return status code + */ +static int gve_setup ( struct gve_nic *gve ) { + unsigned int i; + int rc; + + /* Attempt several times, since the device may decide to add + * in a few spurious resets. + */ + for ( i = 0 ; i < GVE_RESET_MAX_RETRY ; i++ ) { + + /* Reset device */ + if ( ( rc = gve_reset ( gve ) ) != 0 ) + continue; + + /* Enable admin queue */ + gve_admin_enable ( gve ); + + /* Fetch MAC address */ + if ( ( rc = gve_describe ( gve ) ) != 0 ) + continue; + + /* Success */ + return 0; + } + + DBGC ( gve, "GVE %p failed to get device description: %s\n", + gve, strerror ( rc ) ); + return rc; +} + +/** Device startup process descriptor */ +static struct process_descriptor gve_startup_desc = + PROC_DESC_ONCE ( struct gve_nic, startup, gve_startup ); + +/** + * Probe PCI device + * + * @v pci PCI device + * @ret rc Return status code + */ +static int gve_probe ( struct pci_device *pci ) { + struct net_device *netdev; + struct gve_nic *gve; + unsigned long cfg_start; + unsigned long db_start; + unsigned long db_size; + int rc; + + /* Allocate and initialise net device */ + netdev = alloc_etherdev ( sizeof ( *gve ) ); + if ( ! netdev ) { + rc = -ENOMEM; + goto err_alloc; + } + netdev_init ( netdev, &gve_operations ); + gve = netdev->priv; + pci_set_drvdata ( pci, netdev ); + netdev->dev = &pci->dev; + memset ( gve, 0, sizeof ( *gve ) ); + gve->netdev = netdev; + gve->tx.type = &gve_tx_type; + gve->rx.type = &gve_rx_type; + gve->tx.tag = gve->tx_tag; + gve->rx.tag = gve->rx_tag; + process_init_stopped ( &gve->startup, &gve_startup_desc, + &netdev->refcnt ); + timer_init ( &gve->watchdog, gve_watchdog, &netdev->refcnt ); + + /* Fix up PCI device */ + adjust_pci_device ( pci ); + + /* Check PCI revision */ + pci_read_config_byte ( pci, PCI_REVISION, &gve->revision ); + DBGC ( gve, "GVE %p is revision %#02x\n", gve, gve->revision ); + + /* Map configuration registers */ + cfg_start = pci_bar_start ( pci, GVE_CFG_BAR ); + gve->cfg = pci_ioremap ( pci, cfg_start, GVE_CFG_SIZE ); + if ( ! gve->cfg ) { + rc = -ENODEV; + goto err_cfg; + } + + /* Map doorbell registers */ + db_start = pci_bar_start ( pci, GVE_DB_BAR ); + db_size = pci_bar_size ( pci, GVE_DB_BAR ); + gve->db = pci_ioremap ( pci, db_start, db_size ); + if ( ! gve->db ) { + rc = -ENODEV; + goto err_db; + } + + /* Configure DMA */ + gve->dma = &pci->dma; + dma_set_mask_64bit ( gve->dma ); + assert ( netdev->dma == NULL ); + + /* Configure dummy MSI-X interrupt */ + if ( ( rc = pci_msix_enable ( pci, &gve->msix ) ) != 0 ) + goto err_msix; + + /* Allocate admin queue */ + if ( ( rc = gve_admin_alloc ( gve ) ) != 0 ) + goto err_admin; + + /* Set up the device */ + if ( ( rc = gve_setup ( gve ) ) != 0 ) + goto err_setup; + + /* Register network device */ + if ( ( rc = register_netdev ( netdev ) ) != 0 ) + goto err_register_netdev; + + return 0; + + unregister_netdev ( netdev ); + err_register_netdev: + err_setup: + gve_reset ( gve ); + gve_admin_free ( gve ); + err_admin: + pci_msix_disable ( pci, &gve->msix ); + err_msix: + iounmap ( gve->db ); + err_db: + iounmap ( gve->cfg ); + err_cfg: + netdev_nullify ( netdev ); + netdev_put ( netdev ); + err_alloc: + return rc; +} + +/** + * Remove PCI device + * + * @v pci PCI device + */ +static void gve_remove ( struct pci_device *pci ) { + struct net_device *netdev = pci_get_drvdata ( pci ); + struct gve_nic *gve = netdev->priv; + + /* Unregister network device */ + unregister_netdev ( netdev ); + + /* Reset device */ + gve_reset ( gve ); + + /* Free admin queue */ + gve_admin_free ( gve ); + + /* Disable dummy MSI-X interrupt */ + pci_msix_disable ( pci, &gve->msix ); + + /* Unmap registers */ + iounmap ( gve->db ); + iounmap ( gve->cfg ); + + /* Free network device */ + netdev_nullify ( netdev ); + netdev_put ( netdev ); +} + +/** GVE PCI device IDs */ +static struct pci_device_id gve_nics[] = { + PCI_ROM ( 0x1ae0, 0x0042, "gve", "gVNIC", 0 ), +}; + +/** GVE PCI driver */ +struct pci_driver gve_driver __pci_driver = { + .ids = gve_nics, + .id_count = ( sizeof ( gve_nics ) / sizeof ( gve_nics[0] ) ), + .probe = gve_probe, + .remove = gve_remove, +}; |