/*
* (c) Copyright 2010 Stefan Hajnoczi <stefanha@gmail.com>
*
* based on the Etherboot virtio-net driver
*
* (c) Copyright 2008 Bull S.A.S.
*
* Author: Laurent Vivier <Laurent.Vivier@bull.net>
*
* some parts from Linux Virtio PCI driver
*
* Copyright IBM Corp. 2007
* Authors: Anthony Liguori <aliguori@us.ibm.com>
*
* some parts from Linux Virtio Ring
*
* Copyright Rusty Russell IBM Corporation 2007
*
* This work is licensed under the terms of the GNU GPL, version 2 or later.
* See the COPYING file in the top-level directory.
*/
FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
#include <errno.h>
#include <stdlib.h>
#include <unistd.h>
#include <ipxe/list.h>
#include <ipxe/iobuf.h>
#include <ipxe/netdevice.h>
#include <ipxe/pci.h>
#include <ipxe/if_ether.h>
#include <ipxe/ethernet.h>
#include <ipxe/virtio-pci.h>
#include <ipxe/virtio-ring.h>
#include "virtio-net.h"
/*
* Virtio network device driver
*
* Specification:
* http://ozlabs.org/~rusty/virtio-spec/
*
* The virtio network device is supported by Linux virtualization software
* including QEMU/KVM and lguest. This driver supports the virtio over PCI
* transport; virtual machines have one virtio-net PCI adapter per NIC.
*
* Virtio-net is different from hardware NICs because virtio devices
* communicate with the hypervisor via virtqueues, not traditional descriptor
* rings. Virtqueues are unordered queues, they support add_buf() and
* get_buf() operations. To transmit a packet, the driver has to add the
* packet buffer onto the virtqueue. To receive a packet, the driver must
* first add an empty buffer to the virtqueue and then get the filled packet
* buffer on completion.
*
* Virtqueues are an abstraction that is commonly implemented using the vring
* descriptor ring layout. The vring is the actual shared memory structure
* that allows the virtual machine to communicate buffers with the hypervisor.
* Because the vring layout is optimized for flexibility and performance rather
* than space, it is heavy-weight and allocated like traditional descriptor
* rings in the open() function of the driver and not in probe().
*
* There is no true interrupt enable/disable. Virtqueues have callback
* enable/disable flags but these are only hints. The hypervisor may still
* raise an interrupt. Nevertheless, this driver disables callbacks in the
* hopes of avoiding interrupts.
*/
/* Driver types are declared here so virtio-net.h can be easily synced with its
* Linux source.
*/
/* Virtqueue indices */
enum {
RX_INDEX = 0,
TX_INDEX,
QUEUE_NB
};
/** Max number of pending rx packets */
#define NUM_RX_BUF 8
struct virtnet_nic {
/** Base pio register address */
unsigned long ioaddr;
/** 0 for legacy, 1 for virtio 1.0 */
int virtio_version;
/** Virtio 1.0 device data */
struct virtio_pci_modern_device vdev;
/** RX/TX virtqueues */
struct vring_virtqueue *virtqueue;
/** RX packets handed to the NIC waiting to be filled in */
struct list_head rx_iobufs;
/** Pending rx packet count */
unsigned int rx_num_iobufs;
/** Virtio net dummy packet headers */
struct virtio_net_hdr_modern empty_header[QUEUE_NB];
};
/** Add an iobuf to a virtqueue
*
* @v netdev Network device
* @v vq_idx Virtqueue index (RX_INDEX or TX_INDEX)
* @v iobuf I/O buffer
*
* The virtqueue is kicked after the iobuf has been added.
*/
static void virtnet_enqueue_iob ( struct net_device *netdev,
int vq_idx, struct io_buffer *iobuf ) {
struct virtnet_nic *virtnet = netdev->priv;
struct vring_virtqueue *vq = &virtnet->virtqueue[vq_idx];
struct virtio_net_hdr_modern *header = &virtnet->empty_header[vq_idx];
unsigned int out = ( vq_idx == TX_INDEX ) ? 2 : 0;
unsigned int in = ( vq_idx == TX_INDEX ) ? 0 : 2;
size_t header_len = ( virtnet->virtio_version ?
sizeof ( *header ) : sizeof ( header->legacy ) );
struct vring_list list[] = {
{
/* Share a single zeroed virtio net header between all
* packets in a ring. This works because this driver
* does not use any advanced features so none of the
* header fields get used.
*
* Some host implementations (notably Google Compute
* Platform) are known to unconditionally write back
* to header->flags for received packets. Work around
* this by using separate RX and TX headers.
*/
.addr = ( char* ) header,
.length = header_len,
},
{
.addr = ( char* ) iobuf->data,
.length = iob_len ( iobuf ),
},
};
DBGC2 ( virtnet, "VIRTIO-NET %p enqueuing iobuf %p on vq %d\n",
virtnet, iobuf, vq_idx );
vring_add_buf ( vq, list, out, in, iobuf, 0 );
vring_kick ( virtnet->virtio_version ? &virtnet->vdev : NULL,
virtnet->ioaddr, vq, 1 );
}
/** Try to keep rx virtqueue filled with iobufs
*
* @v netdev Network device
*/
static void virtnet_refill_rx_virtqueue ( struct net_device *netdev ) {
struct virtnet_nic *virtnet = netdev->priv;
size_t len = ( netdev->max_pkt_len + 4 /* VLAN */ );
while ( virtnet->rx_num_iobufs < NUM_RX_BUF ) {
struct io_buffer *iobuf;
/* Try to allocate a buffer, stop for now if out of memory */
iobuf = alloc_iob ( len );
if ( ! iobuf )
break;
/* Keep track of iobuf so close() can free it */
list_add ( &iobuf->list, &virtnet->rx_iobufs );
/* Mark packet length until we know the actual size */
iob_put ( iobuf, len );
virtnet_enqueue_iob ( netdev, RX_INDEX, iobuf );
virtnet->rx_num_iobufs++;
}
}
/** Helper to free all virtqueue memory
*
* @v netdev Network device
*/
static void virtnet_free_virtqueues ( struct net_device *netdev ) {
struct virtnet_nic *virtnet = netdev->priv;
int i;
for ( i = 0; i < QUEUE_NB; i++ ) {
virtio_pci_unmap_capability ( &virtnet->virtqueue[i].notification );
vp_free_vq ( &virtnet->virtqueue[i] );
}
free ( virtnet->virtqueue );
virtnet->virtqueue = NULL;
}
/** Open network device, legacy virtio 0.9.5
*
* @v netdev Network device
* @ret rc Return status code
*/
static int virtnet_open_legacy ( struct net_device *netdev ) {
struct virtnet_nic *virtnet = netdev->priv;
unsigned long ioaddr = virtnet->ioaddr;
u32 features;
int i;
/* Reset for sanity */
vp_reset ( ioaddr );
/* Allocate virtqueues */
virtnet->virtqueue = zalloc ( QUEUE_NB *
sizeof ( *virtnet->virtqueue ) );
if ( ! virtnet->virtqueue )
return -ENOMEM;
/* Initialize rx/tx virtqueues */
for ( i = 0; i < QUEUE_NB; i++ ) {
if ( vp_find_vq ( ioaddr, i, &virtnet->virtqueue[i] ) == -1 ) {
DBGC ( virtnet, "VIRTIO-NET %p cannot register queue %d\n",
virtnet, i );
virtnet_free_virtqueues ( netdev );
return -ENOENT;
}
}
/* Initialize rx packets */
INIT_LIST_HEAD ( &virtnet->rx_iobufs );
virtnet->rx_num_iobufs = 0;
virtnet_refill_rx_virtqueue ( netdev );
/* Disable interrupts before starting */
netdev_irq ( netdev, 0 );
/* Driver is ready */
features = vp_get_features ( ioaddr );
vp_set_features ( ioaddr, features & ( ( 1 << VIRTIO_NET_F_MAC ) |
( 1 << VIRTIO_NET_F_MTU ) ) );
vp_set_status ( ioaddr, VIRTIO_CONFIG_S_DRIVER | VIRTIO_CONFIG_S_DRIVER_OK );
return 0;
}
/** Open network device, modern virtio 1.0
*
* @v netdev Network device
* @ret rc Return status code
*/
static int virtnet_open_modern ( struct net_device *netdev ) {
struct virtnet_nic *virtnet = netdev->priv;
u64 features;
u8 status;
/* Negotiate features */
features = vpm_get_features ( &virtnet->vdev );
if ( ! ( features & VIRTIO_F_VERSION_1 ) ) {
vpm_add_status ( &virtnet->vdev, VIRTIO_CONFIG_S_FAILED );
return -EINVAL;
}
vpm_set_features ( &virtnet->vdev, features & (
( 1ULL << VIRTIO_NET_F_MAC ) |
( 1ULL << VIRTIO_NET_F_MTU ) |
( 1ULL << VIRTIO_F_VERSION_1 ) |
( 1ULL << VIRTIO_F_ANY_LAYOUT ) |
( 1ULL << VIRTIO_F_IOMMU_PLATFORM ) ) );
vpm_add_status ( &virtnet->vdev, VIRTIO_CONFIG_S_FEATURES_OK );
status = vpm_get_status ( &virtnet->vdev );
if ( ! ( status & VIRTIO_CONFIG_S_FEATURES_OK ) ) {
DBGC ( virtnet, "VIRTIO-NET %p device didn't accept features\n",
virtnet );
vpm_add_status ( &virtnet->vdev, VIRTIO_CONFIG_S_FAILED );
return -EINVAL;
}
/* Allocate virtqueues */
virtnet->virtqueue = zalloc ( QUEUE_NB *
sizeof ( *virtnet->virtqueue ) );
if ( ! virtnet->virtqueue ) {
vpm_add_status ( &virtnet->vdev, VIRTIO_CONFIG_S_FAILED );
return -ENOMEM;
}
/* Initialize rx/tx virtqueues */
if ( vpm_find_vqs ( &virtnet->vdev, QUEUE_NB, virtnet->virtqueue ) ) {
DBGC ( virtnet, "VIRTIO-NET %p cannot register queues\n",
virtnet );
virtnet_free_virtqueues ( netdev );
vpm_add_status ( &virtnet->vdev, VIRTIO_CONFIG_S_FAILED );
return -ENOENT;
}
/* Disable interrupts before starting */
netdev_irq ( netdev, 0 );
vpm_add_status ( &virtnet->vdev, VIRTIO_CONFIG_S_DRIVER_OK );
/* Initialize rx packets */
INIT_LIST_HEAD ( &virtnet->rx_iobufs );
virtnet->rx_num_iobufs = 0;
virtnet_refill_rx_virtqueue ( netdev );
return 0;
}
/** Open network device
*
* @v netdev Network device
* @ret rc Return status code
*/
static int virtnet_open ( struct net_device *netdev ) {
struct virtnet_nic *virtnet = netdev->priv;
if ( virtnet->virtio_version ) {
return virtnet_open_modern ( netdev );
} else {
return virtnet_open_legacy ( netdev );
}
}
/** Close network device
*
* @v netdev Network device
*/
static void virtnet_close ( struct net_device *netdev ) {
struct virtnet_nic *virtnet = netdev->priv;
struct io_buffer *iobuf;
struct io_buffer *next_iobuf;
if ( virtnet->virtio_version ) {
vpm_reset ( &virtnet->vdev );
} else {
vp_reset ( virtnet->ioaddr );
}
/* Virtqueues can be freed now that NIC is reset */
virtnet_free_virtqueues ( netdev );
/* Free rx iobufs */
list_for_each_entry_safe ( iobuf, next_iobuf, &virtnet->rx_iobufs, list ) {
free_iob ( iobuf );
}
INIT_LIST_HEAD ( &virtnet->rx_iobufs );
virtnet->rx_num_iobufs = 0;
}
/** Transmit packet
*
* @v netdev Network device
* @v iobuf I/O buffer
* @ret rc Return status code
*/
static int virtnet_transmit ( struct net_device *netdev,
struct io_buffer *iobuf ) {
virtnet_enqueue_iob ( netdev, TX_INDEX, iobuf );
return 0;
}
/** Complete packet transmission
*
* @v netdev Network device
*/
static void virtnet_process_tx_packets ( struct net_device *netdev ) {
struct virtnet_nic *virtnet = netdev->priv;
struct vring_virtqueue *tx_vq = &virtnet->virtqueue[TX_INDEX];
while ( vring_more_used ( tx_vq ) ) {
struct io_buffer *iobuf = vring_get_buf ( tx_vq, NULL );
DBGC2 ( virtnet, "VIRTIO-NET %p tx complete iobuf %p\n",
virtnet, iobuf );
netdev_tx_complete ( netdev, iobuf );
}
}
/** Complete packet reception
*
* @v netdev Network device
*/
static void virtnet_process_rx_packets ( struct net_device *netdev ) {
struct virtnet_nic *virtnet = netdev->priv;
struct vring_virtqueue *rx_vq = &virtnet->virtqueue[RX_INDEX];
while ( vring_more_used ( rx_vq ) ) {
unsigned int len;
struct io_buffer *iobuf = vring_get_buf ( rx_vq, &len );
/* Release ownership of iobuf */
list_del ( &iobuf->list );
virtnet->rx_num_iobufs--;
/* Update iobuf length */
iob_unput ( iobuf, iob_len ( iobuf ) );
iob_put ( iobuf, len - sizeof ( struct virtio_net_hdr ) );
DBGC2 ( virtnet, "VIRTIO-NET %p rx complete iobuf %p len %zd\n",
virtnet, iobuf, iob_len ( iobuf ) );
/* Pass completed packet to the network stack */
netdev_rx ( netdev, iobuf );
}
virtnet_refill_rx_virtqueue ( netdev );
}
/** Poll for completed and received packets
*
* @v netdev Network device
*/
static void virtnet_poll ( struct net_device *netdev ) {
struct virtnet_nic *virtnet = netdev->priv;
/* Acknowledge interrupt. This is necessary for UNDI operation and
* interrupts that are raised despite VRING_AVAIL_F_NO_INTERRUPT being
* set (that flag is just a hint and the hypervisor does not have to
* honor it).
*/
if ( virtnet->virtio_version ) {
vpm_get_isr ( &virtnet->vdev );
} else {
vp_get_isr ( virtnet->ioaddr );
}
virtnet_process_tx_packets ( netdev );
virtnet_process_rx_packets ( netdev );
}
/** Enable or disable interrupts
*
* @v netdev Network device
* @v enable Interrupts should be enabled
*/
static void virtnet_irq ( struct net_device *netdev, int enable ) {
struct virtnet_nic *virtnet = netdev->priv;
int i;
for ( i = 0; i < QUEUE_NB; i++ ) {
if ( enable )
vring_enable_cb ( &virtnet->virtqueue[i] );
else
vring_disable_cb ( &virtnet->virtqueue[i] );
}
}
/** virtio-net device operations */
static struct net_device_operations virtnet_operations = {
.open = virtnet_open,
.close = virtnet_close,
.transmit = virtnet_transmit,
.poll = virtnet_poll,
.irq = virtnet_irq,
};
/**
* Probe PCI device, legacy virtio 0.9.5
*
* @v pci PCI device
* @ret rc Return status code
*/
static int virtnet_probe_legacy ( struct pci_device *pci ) {
unsigned long ioaddr = pci->ioaddr;
struct net_device *netdev;
struct virtnet_nic *virtnet;
u32 features;
u16 mtu;
int rc;
/* Allocate and hook up net device */
netdev = alloc_etherdev ( sizeof ( *virtnet ) );
if ( ! netdev )
return -ENOMEM;
netdev_init ( netdev, &virtnet_operations );
virtnet = netdev->priv;
virtnet->ioaddr = ioaddr;
pci_set_drvdata ( pci, netdev );
netdev->dev = &pci->dev;
DBGC ( virtnet, "VIRTIO-NET %p busaddr=%s ioaddr=%#lx irq=%d\n",
virtnet, pci->dev.name, ioaddr, pci->irq );
/* Enable PCI bus master and reset NIC */
adjust_pci_device ( pci );
vp_reset ( ioaddr );
/* Load MAC address and MTU */
features = vp_get_features ( ioaddr );
if ( features & ( 1 << VIRTIO_NET_F_MAC ) ) {
vp_get ( ioaddr, offsetof ( struct virtio_net_config, mac ),
netdev->hw_addr, ETH_ALEN );
DBGC ( virtnet, "VIRTIO-NET %p mac=%s\n", virtnet,
eth_ntoa ( netdev->hw_addr ) );
}
if ( features & ( 1ULL << VIRTIO_NET_F_MTU ) ) {
vp_get ( ioaddr, offsetof ( struct virtio_net_config, mtu ),
&mtu, sizeof ( mtu ) );
DBGC ( virtnet, "VIRTIO-NET %p mtu=%d\n", virtnet, mtu );
netdev->max_pkt_len = ( mtu + ETH_HLEN );
netdev->mtu = mtu;
}
/* Register network device */
if ( ( rc = register_netdev ( netdev ) ) != 0 )
goto err_register_netdev;
/* Mark link as up, control virtqueue is not used */
netdev_link_up ( netdev );
return 0;
unregister_netdev ( netdev );
err_register_netdev:
vp_reset ( ioaddr );
netdev_nullify ( netdev );
netdev_put ( netdev );
return rc;
}
/**
* Probe PCI device, modern virtio 1.0
*
* @v pci PCI device
* @v found_dev Set to non-zero if modern device was found (probe may still fail)
* @ret rc Return status code
*/
static int virtnet_probe_modern ( struct pci_device *pci, int *found_dev ) {
struct net_device *netdev;
struct virtnet_nic *virtnet;
u64 features;
u16 mtu;
int rc, common, isr, notify, config, device;
common = virtio_pci_find_capability ( pci, VIRTIO_PCI_CAP_COMMON_CFG );
if ( ! common ) {
DBG ( "Common virtio capability not found!\n" );
return -ENODEV;
}
*found_dev = 1;
isr = virtio_pci_find_capability ( pci, VIRTIO_PCI_CAP_ISR_CFG );
notify = virtio_pci_find_capability ( pci, VIRTIO_PCI_CAP_NOTIFY_CFG );
config = virtio_pci_find_capability ( pci, VIRTIO_PCI_CAP_PCI_CFG );
if ( ! isr || ! notify || ! config ) {
DBG ( "Missing virtio capabilities %i/%i/%i/%i\n",
common, isr, notify, config );
return -EINVAL;
}
device = virtio_pci_find_capability ( pci, VIRTIO_PCI_CAP_DEVICE_CFG );
/* Allocate and hook up net device */
netdev = alloc_etherdev ( sizeof ( *virtnet ) );
if ( ! netdev )
return -ENOMEM;
netdev_init ( netdev, &virtnet_operations );
virtnet = netdev->priv;
pci_set_drvdata ( pci, netdev );
netdev->dev = &pci->dev;
DBGC ( virtnet, "VIRTIO-NET modern %p busaddr=%s irq=%d\n",
virtnet, pci->dev.name, pci->irq );
virtnet->vdev.pci = pci;
rc = virtio_pci_map_capability ( pci, common,
sizeof ( struct virtio_pci_common_cfg ), 4,
0, sizeof ( struct virtio_pci_common_cfg ),
&virtnet->vdev.common );
if ( rc )
goto err_map_common;
rc = virtio_pci_map_capability ( pci, isr, sizeof ( u8 ), 1,
0, 1,
&virtnet->vdev.isr );
if ( rc )
goto err_map_isr;
virtnet->vdev.notify_cap_pos = notify;
virtnet->vdev.cfg_cap_pos = config;
/* Map the device capability */
if ( device ) {
rc = virtio_pci_map_capability ( pci, device,
0, 4, 0, sizeof ( struct virtio_net_config ),
&virtnet->vdev.device );
if ( rc )
goto err_map_device;
}
/* Enable the PCI device */
adjust_pci_device ( pci );
/* Reset the device and set initial status bits */
vpm_reset ( &virtnet->vdev );
vpm_add_status ( &virtnet->vdev, VIRTIO_CONFIG_S_ACKNOWLEDGE );
vpm_add_status ( &virtnet->vdev, VIRTIO_CONFIG_S_DRIVER );
/* Load MAC address and MTU */
if ( device ) {
features = vpm_get_features ( &virtnet->vdev );
if ( features & ( 1ULL << VIRTIO_NET_F_MAC ) ) {
vpm_get ( &virtnet->vdev,
offsetof ( struct virtio_net_config, mac ),
netdev->hw_addr, ETH_ALEN );
DBGC ( virtnet, "VIRTIO-NET %p mac=%s\n", virtnet,
eth_ntoa ( netdev->hw_addr ) );
}
if ( features & ( 1ULL << VIRTIO_NET_F_MTU ) ) {
vpm_get ( &virtnet->vdev,
offsetof ( struct virtio_net_config, mtu ),
&mtu, sizeof ( mtu ) );
DBGC ( virtnet, "VIRTIO-NET %p mtu=%d\n", virtnet,
mtu );
netdev->max_pkt_len = ( mtu + ETH_HLEN );
}
}
/* We need a valid MAC address */
if ( ! is_valid_ether_addr ( netdev->hw_addr ) ) {
rc = -EADDRNOTAVAIL;
goto err_mac_address;
}
/* Register network device */
if ( ( rc = register_netdev ( netdev ) ) != 0 )
goto err_register_netdev;
/* Mark link as up, control virtqueue is not used */
netdev_link_up ( netdev );
virtnet->virtio_version = 1;
return 0;
unregister_netdev ( netdev );
err_register_netdev:
err_mac_address:
vpm_reset ( &virtnet->vdev );
netdev_nullify ( netdev );
netdev_put ( netdev );
virtio_pci_unmap_capability ( &virtnet->vdev.device );
err_map_device:
virtio_pci_unmap_capability ( &virtnet->vdev.isr );
err_map_isr:
virtio_pci_unmap_capability ( &virtnet->vdev.common );
err_map_common:
return rc;
}
/**
* Probe PCI device
*
* @v pci PCI device
* @ret rc Return status code
*/
static int virtnet_probe ( struct pci_device *pci ) {
int found_modern = 0;
int rc = virtnet_probe_modern ( pci, &found_modern );
if ( ! found_modern && pci->device < 0x1040 ) {
/* fall back to the legacy probe */
rc = virtnet_probe_legacy ( pci );
}
return rc;
}
/**
* Remove device
*
* @v pci PCI device
*/
static void virtnet_remove ( struct pci_device *pci ) {
struct net_device *netdev = pci_get_drvdata ( pci );
struct virtnet_nic *virtnet = netdev->priv;
virtio_pci_unmap_capability ( &virtnet->vdev.device );
virtio_pci_unmap_capability ( &virtnet->vdev.isr );
virtio_pci_unmap_capability ( &virtnet->vdev.common );
unregister_netdev ( netdev );
netdev_nullify ( netdev );
netdev_put ( netdev );
}
static struct pci_device_id virtnet_nics[] = {
PCI_ROM(0x1af4, 0x1000, "virtio-net", "Virtio Network Interface", 0),
PCI_ROM(0x1af4, 0x1041, "virtio-net", "Virtio Network Interface 1.0", 0),
};
struct pci_driver virtnet_driver __pci_driver = {
.ids = virtnet_nics,
.id_count = ( sizeof ( virtnet_nics ) / sizeof ( virtnet_nics[0] ) ),
.probe = virtnet_probe,
.remove = virtnet_remove,
};
