[pci] Add support for PCI MSI-X interrupts

The Intel 40 Gigabit Ethernet virtual functions support only MSI-X
interrupts, and will write back completed interrupt descriptors only
when the device attempts to raise an interrupt (or when a complete
cacheline of receive descriptors has been completed).

We cannot actually use MSI-X interrupts within iPXE, since we never
have ownership of the APIC.  However, an MSI-X interrupt is
fundamentally just a DMA write of a single dword to an arbitrary
address.  We can therefore configure the device to "raise" an
interrupt by writing a meaningless value to an otherwise unused memory
location: this is sufficient to trigger the receive descriptor
writeback logic.

Signed-off-by: Michael Brown <mcb30@ipxe.org>

4 files changed, 340 insertions, 0 deletions

diff --git a/src/drivers/bus/pcimsix.c b/src/drivers/bus/pcimsix.c
new file mode 100644
index 00000000..80893c41
--- /dev/null
+++ b/src/drivers/bus/pcimsix.c
@@ -0,0 +1,251 @@
+/*
+ * Copyright (C) 2019 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <errno.h>
+#include <assert.h>
+#include <ipxe/pci.h>
+#include <ipxe/pcimsix.h>
+
+/** @file
+ *
+ * PCI MSI-X interrupts
+ *
+ */
+
+/**
+ * Get MSI-X descriptor name (for debugging)
+ *
+ * @v cfg		Configuration space offset
+ * @ret name		Descriptor name
+ */
+static const char * pci_msix_name ( unsigned int cfg ) {
+
+	switch ( cfg ) {
+	case PCI_MSIX_DESC_TABLE:	return "table";
+	case PCI_MSIX_DESC_PBA:		return "PBA";
+	default:			return "<UNKNOWN>";
+	}
+}
+
+/**
+ * Map MSI-X BAR portion
+ *
+ * @v pci		PCI device
+ * @v msix		MSI-X capability
+ * @v cfg		Configuration space offset
+ * @ret io		I/O address
+ */
+static void * pci_msix_ioremap ( struct pci_device *pci, struct pci_msix *msix,
+				 unsigned int cfg ) {
+	uint32_t desc;
+	unsigned int bar;
+	unsigned long start;
+	unsigned long offset;
+	unsigned long base;
+	void *io;
+
+	/* Read descriptor */
+	pci_read_config_dword ( pci, ( msix->cap + cfg ), &desc );
+
+	/* Get BAR */
+	bar = PCI_MSIX_DESC_BIR ( desc );
+	offset = PCI_MSIX_DESC_OFFSET ( desc );
+	start = pci_bar_start ( pci, PCI_BASE_ADDRESS ( bar ) );
+	if ( ! start ) {
+		DBGC ( msix, "MSI-X %p %s could not find BAR%d\n",
+		       msix, pci_msix_name ( cfg ), bar );
+		return NULL;
+	}
+	base = ( start + offset );
+	DBGC ( msix, "MSI-X %p %s at %#08lx (BAR%d+%#lx)\n",
+	       msix, pci_msix_name ( cfg ), base, bar, offset );
+
+	/* Map BAR portion */
+	io = ioremap ( ( start + offset ), PCI_MSIX_LEN );
+	if ( ! io ) {
+		DBGC ( msix, "MSI-X %p %s could not map %#08lx\n",
+		       msix, pci_msix_name ( cfg ), base );
+		return NULL;
+	}
+
+	return io;
+}
+
+/**
+ * Enable MSI-X interrupts
+ *
+ * @v pci		PCI device
+ * @v msix		MSI-X capability
+ * @ret rc		Return status code
+ */
+int pci_msix_enable ( struct pci_device *pci, struct pci_msix *msix ) {
+	uint16_t ctrl;
+	int rc;
+
+	/* Locate capability */
+	msix->cap = pci_find_capability ( pci, PCI_CAP_ID_MSIX );
+	if ( ! msix->cap ) {
+		DBGC ( msix, "MSI-X %p found no MSI-X capability in "
+		       PCI_FMT "\n", msix, PCI_ARGS ( pci ) );
+		rc = -ENOENT;
+		goto err_cap;
+	}
+
+	/* Extract interrupt count */
+	pci_read_config_word ( pci, ( msix->cap + PCI_MSIX_CTRL ), &ctrl );
+	msix->count = ( PCI_MSIX_CTRL_SIZE ( ctrl ) + 1 );
+	DBGC ( msix, "MSI-X %p has %d vectors for " PCI_FMT "\n",
+	       msix, msix->count, PCI_ARGS ( pci ) );
+
+	/* Map MSI-X table */
+	msix->table = pci_msix_ioremap ( pci, msix, PCI_MSIX_DESC_TABLE );
+	if ( ! msix->table ) {
+		rc = -ENOENT;
+		goto err_table;
+	}
+
+	/* Map pending bit array */
+	msix->pba = pci_msix_ioremap ( pci, msix, PCI_MSIX_DESC_PBA );
+	if ( ! msix->pba ) {
+		rc = -ENOENT;
+		goto err_pba;
+	}
+
+	/* Enable MSI-X */
+	ctrl &= ~PCI_MSIX_CTRL_MASK;
+	ctrl |= PCI_MSIX_CTRL_ENABLE;
+	pci_write_config_word ( pci, ( msix->cap + PCI_MSIX_CTRL ), ctrl );
+
+	return 0;
+
+	iounmap ( msix->pba );
+ err_pba:
+	iounmap ( msix->table );
+ err_table:
+ err_cap:
+	return rc;
+}
+
+/**
+ * Disable MSI-X interrupts
+ *
+ * @v pci		PCI device
+ * @v msix		MSI-X capability
+ */
+void pci_msix_disable ( struct pci_device *pci, struct pci_msix *msix ) {
+	uint16_t ctrl;
+
+	/* Disable MSI-X */
+	pci_read_config_word ( pci, ( msix->cap + PCI_MSIX_CTRL ), &ctrl );
+	ctrl &= ~PCI_MSIX_CTRL_ENABLE;
+	pci_write_config_word ( pci, ( msix->cap + PCI_MSIX_CTRL ), ctrl );
+
+	/* Unmap pending bit array */
+	iounmap ( msix->pba );
+
+	/* Unmap MSI-X table */
+	iounmap ( msix->table );
+}
+
+/**
+ * Map MSI-X interrupt vector
+ *
+ * @v msix		MSI-X capability
+ * @v vector		MSI-X vector
+ * @v address		Message address
+ * @v data		Message data
+ */
+void pci_msix_map ( struct pci_msix *msix, unsigned int vector,
+		    physaddr_t address, uint32_t data ) {
+	void *base;
+
+	/* Sanity check */
+	assert ( vector < msix->count );
+
+	/* Map interrupt vector */
+	base = ( msix->table + PCI_MSIX_VECTOR ( vector ) );
+	writel ( ( address & 0xffffffffUL ), ( base + PCI_MSIX_ADDRESS_LO ) );
+	if ( sizeof ( address ) > sizeof ( uint32_t ) ) {
+		writel ( ( ( ( uint64_t ) address ) >> 32 ),
+			 ( base + PCI_MSIX_ADDRESS_HI ) );
+	} else {
+		writel ( 0, ( base + PCI_MSIX_ADDRESS_HI ) );
+	}
+	writel ( data, ( base + PCI_MSIX_DATA ) );
+}
+
+/**
+ * Control MSI-X interrupt vector
+ *
+ * @v msix		MSI-X capability
+ * @v vector		MSI-X vector
+ * @v mask		Control mask
+ */
+void pci_msix_control ( struct pci_msix *msix, unsigned int vector,
+			uint32_t mask ) {
+	void *base;
+	uint32_t ctrl;
+
+	/* Mask/unmask interrupt vector */
+	base = ( msix->table + PCI_MSIX_VECTOR ( vector ) );
+	ctrl = readl ( base + PCI_MSIX_CONTROL );
+	ctrl &= ~PCI_MSIX_CONTROL_MASK;
+	ctrl |= mask;
+	writel ( ctrl, ( base + PCI_MSIX_CONTROL ) );
+}
+
+/**
+ * Dump MSI-X interrupt state (for debugging)
+ *
+ * @v msix		MSI-X capability
+ * @v vector		MSI-X vector
+ */
+void pci_msix_dump ( struct pci_msix *msix, unsigned int vector ) {
+	void *base;
+	uint32_t address_hi;
+	uint32_t address_lo;
+	physaddr_t address;
+	uint32_t data;
+	uint32_t ctrl;
+	uint32_t pba;
+
+	/* Do nothing in non-debug builds */
+	if ( ! DBG_LOG )
+		return;
+
+	/* Mask/unmask interrupt vector */
+	base = ( msix->table + PCI_MSIX_VECTOR ( vector ) );
+	address_hi = readl ( base + PCI_MSIX_ADDRESS_HI );
+	address_lo = readl ( base + PCI_MSIX_ADDRESS_LO );
+	data = readl ( base + PCI_MSIX_DATA );
+	ctrl = readl ( base + PCI_MSIX_CONTROL );
+	pba = readl ( msix->pba );
+	address = ( ( ( ( uint64_t ) address_hi ) << 32 ) | address_lo );
+	DBGC ( msix, "MSI-X %p vector %d %#08x => %#08lx%s%s\n",
+	       msix, vector, data, address,
+	       ( ( ctrl & PCI_MSIX_CONTROL_MASK ) ? " (masked)" : "" ),
+	       ( ( pba & ( 1 << vector ) ) ? " (pending)" : "" ) );
+}
diff --git a/src/include/ipxe/errfile.h b/src/include/ipxe/errfile.h
index ce67fc66..02e13d11 100644
--- a/src/include/ipxe/errfile.h
+++ b/src/include/ipxe/errfile.h
@@ -205,6 +205,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_ena		     ( ERRFILE_DRIVER | 0x00c90000 )
 #define ERRFILE_icplus		     ( ERRFILE_DRIVER | 0x00ca0000 )
 #define ERRFILE_intelxl		     ( ERRFILE_DRIVER | 0x00cb0000 )
+#define ERRFILE_pcimsix		     ( ERRFILE_DRIVER | 0x00cc0000 )
 
 #define ERRFILE_aoe			( ERRFILE_NET | 0x00000000 )
 #define ERRFILE_arp			( ERRFILE_NET | 0x00010000 )
diff --git a/src/include/ipxe/pci.h b/src/include/ipxe/pci.h
index ddd8c8d1..272c4c06 100644
--- a/src/include/ipxe/pci.h
+++ b/src/include/ipxe/pci.h
@@ -94,6 +94,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define PCI_CAP_ID_VPD			0x03	/**< Vital product data */
 #define PCI_CAP_ID_VNDR			0x09	/**< Vendor-specific */
 #define PCI_CAP_ID_EXP			0x10	/**< PCI Express */
+#define PCI_CAP_ID_MSIX			0x11	/**< MSI-X */
 #define PCI_CAP_ID_EA			0x14	/**< Enhanced Allocation */
 
 /** Next capability */
@@ -109,6 +110,16 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define PCI_EXP_DEVCTL		0x08
 #define PCI_EXP_DEVCTL_FLR		0x8000	/**< Function level reset */
 
+/** MSI-X interrupts */
+#define PCI_MSIX_CTRL		0x02
+#define PCI_MSIX_CTRL_ENABLE		0x8000	/**< Enable MSI-X */
+#define PCI_MSIX_CTRL_MASK		0x4000	/**< Mask all interrupts */
+#define PCI_MSIX_CTRL_SIZE(x)	( (x) & 0x07ff ) /**< Table size */
+#define PCI_MSIX_DESC_TABLE	0x04
+#define PCI_MSIX_DESC_PBA	0x08
+#define PCI_MSIX_DESC_BIR(x)	( (x) & 0x00000007 ) /**< BAR index */
+#define PCI_MSIX_DESC_OFFSET(x)	( (x) & 0xfffffff8 ) /**< BAR offset */
+
 /** Uncorrectable error status */
 #define PCI_ERR_UNCOR_STATUS	0x04
 
diff --git a/src/include/ipxe/pcimsix.h b/src/include/ipxe/pcimsix.h
new file mode 100644
index 00000000..aa2aaf01
--- /dev/null
+++ b/src/include/ipxe/pcimsix.h
@@ -0,0 +1,77 @@
+#ifndef _IPXE_PCIMSIX_H
+#define _IPXE_PCIMSIX_H
+
+/** @file
+ *
+ * PCI MSI-X interrupts
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/pci.h>
+
+/** MSI-X BAR mapped length */
+#define PCI_MSIX_LEN 0x1000
+
+/** MSI-X vector offset */
+#define PCI_MSIX_VECTOR(n) ( (n) * 0x10 )
+
+/** MSI-X vector address low 32 bits */
+#define PCI_MSIX_ADDRESS_LO 0x0
+
+/** MSI-X vector address high 32 bits */
+#define PCI_MSIX_ADDRESS_HI 0x4
+
+/** MSI-X vector data */
+#define PCI_MSIX_DATA 0x8
+
+/** MSI-X vector control */
+#define PCI_MSIX_CONTROL 0xc
+#define PCI_MSIX_CONTROL_MASK 0x00000001	/**< Vector is masked */
+
+/** PCI MSI-X capability */
+struct pci_msix {
+	/** Capability offset */
+	unsigned int cap;
+	/** Number of vectors */
+	unsigned int count;
+	/** MSI-X table */
+	void *table;
+	/** Pending bit array */
+	void *pba;
+};
+
+extern int pci_msix_enable ( struct pci_device *pci, struct pci_msix *msix );
+extern void pci_msix_disable ( struct pci_device *pci, struct pci_msix *msix );
+extern void pci_msix_map ( struct pci_msix *msix, unsigned int vector,
+			   physaddr_t address, uint32_t data );
+extern void pci_msix_control ( struct pci_msix *msix, unsigned int vector,
+			       uint32_t mask );
+extern void pci_msix_dump ( struct pci_msix *msix, unsigned int vector );
+
+/**
+ * Mask MSI-X interrupt vector
+ *
+ * @v msix		MSI-X capability
+ * @v vector		MSI-X vector
+ */
+static inline __attribute__ (( always_inline )) void
+pci_msix_mask ( struct pci_msix *msix, unsigned int vector ) {
+
+	pci_msix_control ( msix, vector, PCI_MSIX_CONTROL_MASK );
+}
+
+/**
+ * Unmask MSI-X interrupt vector
+ *
+ * @v msix		MSI-X capability
+ * @v vector		MSI-X vector
+ */
+static inline __attribute__ (( always_inline )) void
+pci_msix_unmask ( struct pci_msix *msix, unsigned int vector ) {
+
+	pci_msix_control ( msix, vector, 0 );
+}
+
+#endif /* _IPXE_PCIMSIX_H */