Merge branch 'aqc1xx' into openslx

190 files changed, 11124 insertions, 1530 deletions

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 232375ad..c6988a11 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -12,7 +12,7 @@ jobs:
         run: |
           sudo chown $(id -un) /var/cache/apt/archives
       - name: Cache packages
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /var/cache/apt/archives/*.deb
           key: apt-cache-${{ github.run_id }}-${{ github.run_attempt }}
@@ -32,14 +32,14 @@ jobs:
     needs: cache
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Cache permissions
         run: |
           sudo chown $(id -un) /var/cache/apt/archives
       - name: Cache packages
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: /var/cache/apt/archives/*.deb
           key: apt-cache-${{ github.run_id }}-${{ github.run_attempt }}
@@ -68,14 +68,14 @@ jobs:
     needs: cache
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Cache permissions
         run: |
           sudo chown $(id -un) /var/cache/apt/archives
       - name: Cache packages
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: /var/cache/apt/archives/*.deb
           key: apt-cache-${{ github.run_id }}-${{ github.run_attempt }}
@@ -97,14 +97,14 @@ jobs:
     needs: cache
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Cache permissions
         run: |
           sudo chown $(id -un) /var/cache/apt/archives
       - name: Cache packages
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: /var/cache/apt/archives/*.deb
           key: apt-cache-${{ github.run_id }}-${{ github.run_attempt }}
diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml
index ba868180..4c3e2e48 100644
--- a/.github/workflows/coverity.yml
+++ b/.github/workflows/coverity.yml
@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Download Coverity Scan
         run: |
           curl --form token=${{ secrets.COVERITY_SCAN_TOKEN }} \
diff --git a/contrib/cloud/aws-import b/contrib/cloud/aws-import
index b9a350f6..ace00587 100755
--- a/contrib/cloud/aws-import
+++ b/contrib/cloud/aws-import
@@ -46,11 +46,19 @@ def create_snapshot(region, description, image):
     return snapshot_id
 
 
-def import_image(region, name, architecture, image, public):
+def import_image(region, name, architecture, image, public, overwrite):
     """Import an AMI image"""
     client = boto3.client('ec2', region_name=region)
     resource = boto3.resource('ec2', region_name=region)
     description = '%s (%s)' % (name, architecture)
+    images = client.describe_images(Filters=[{'Name': 'name',
+                                              'Values': [description]}])
+    if overwrite and images['Images']:
+        images = images['Images'][0]
+        image_id = images['ImageId']
+        snapshot_id = images['BlockDeviceMappings'][0]['Ebs']['SnapshotId']
+        resource.Image(image_id).deregister()
+        resource.Snapshot(snapshot_id).delete()
     snapshot_id = create_snapshot(region=region, description=description,
                                   image=image)
     client.get_waiter('snapshot_completed').wait(SnapshotIds=[snapshot_id])
@@ -88,6 +96,8 @@ parser.add_argument('--name', '-n',
                     help="Image name")
 parser.add_argument('--public', '-p', action='store_true',
                     help="Make image public")
+parser.add_argument('--overwrite', action='store_true',
+                    help="Overwrite any existing image with same name")
 parser.add_argument('--region', '-r', action='append',
                     help="AWS region(s)")
 parser.add_argument('--wiki', '-w', metavar='FILE',
@@ -115,7 +125,8 @@ with ThreadPoolExecutor(max_workers=len(imports)) as executor:
                                name=args.name,
                                architecture=architectures[image],
                                image=image,
-                               public=args.public): (region, image)
+                               public=args.public,
+                               overwrite=args.overwrite): (region, image)
                for region, image in imports}
     results = {futures[future]: future.result()
                for future in as_completed(futures)}
diff --git a/contrib/cloud/aws-int13con b/contrib/cloud/aws-int13con
new file mode 100755
index 00000000..b79b4065
--- /dev/null
+++ b/contrib/cloud/aws-int13con
@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+
+import argparse
+
+import boto3
+
+BLOCKSIZE = 512 * 1024
+
+IPXELOG_OFFSET = 16 * 1024
+
+IPXELOG_MAGIC = b'iPXE LOG'
+
+
+def create_snapshot(region, instance_id):
+    """Create root volume snapshot"""
+    client = boto3.client('ec2', region_name=region)
+    resource = boto3.resource('ec2', region_name=region)
+    instance = resource.Instance(instance_id)
+    volumes = list(instance.volumes.all())
+    snapshot = volumes[0].create_snapshot()
+    snapshot.wait_until_completed()
+    return snapshot.id
+
+
+def get_snapshot_block(region, snapshot_id, index):
+    """Get block content from snapshot"""
+    client = boto3.client('ebs', region_name=region)
+    blocks = client.list_snapshot_blocks(SnapshotId=snapshot_id,
+                                         StartingBlockIndex=index)
+    token = blocks['Blocks'][0]['BlockToken']
+    block = client.get_snapshot_block(SnapshotId=snapshot_id,
+                                      BlockIndex=index,
+                                      BlockToken=token)
+    return block['BlockData'].read()
+
+
+def get_block0_content(region, instance_id):
+    """Get content of root volume block zero from instance"""
+    client = boto3.client('ec2', region_name=region)
+    resource = boto3.resource('ec2', region_name=region)
+    snapshot_id = create_snapshot(region, instance_id)
+    block = get_snapshot_block(region, snapshot_id, 0)
+    resource.Snapshot(snapshot_id).delete()
+    return block
+
+
+def get_int13con_output(region, instance_id):
+    """Get INT13 console output"""
+    block = get_block0_content(region, instance_id)
+    logpart = block[IPXELOG_OFFSET:]
+    magic = logpart[:len(IPXELOG_MAGIC)]
+    if magic != IPXELOG_MAGIC:
+        raise ValueError("Invalid log magic signature")
+    log = logpart[len(IPXELOG_MAGIC):].split(b'\0')[0]
+    return log.decode()
+
+
+# Parse command-line arguments
+parser = argparse.ArgumentParser(description="Get AWS INT13 console output")
+parser.add_argument('--region', '-r', help="AWS region")
+parser.add_argument('id', help="Instance ID")
+args = parser.parse_args()
+
+# Get console output from INT13CON partition
+output = get_int13con_output(args.region, args.id)
+
+# Print console output
+print(output)
diff --git a/src/.gitignore b/src/.gitignore
index cc8e33e2..4e4f00c8 100644
--- a/src/.gitignore
+++ b/src/.gitignore
@@ -1,4 +1,4 @@
 .toolcheck
 .echocheck
 TAGS*
-bin*
+bin-*
diff --git a/src/Makefile b/src/Makefile
index bc82cc6f..95719628 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -77,6 +77,7 @@ SRCDIRS		+= drivers/net/efi
 SRCDIRS		+= drivers/net/tg3
 SRCDIRS		+= drivers/net/bnxt
 SRCDIRS		+= drivers/net/sfc
+SRCDIRS		+= drivers/net/marvell
 SRCDIRS		+= drivers/block
 SRCDIRS		+= drivers/nvs
 SRCDIRS		+= drivers/bitbash
diff --git a/src/arch/arm/interface/efi/efiarm_nap.c b/src/arch/arm/interface/efi/efiarm_nap.c
index 9ed638e9..fba7a5d8 100644
--- a/src/arch/arm/interface/efi/efiarm_nap.c
+++ b/src/arch/arm/interface/efi/efiarm_nap.c
@@ -46,8 +46,12 @@ static void efiarm_cpu_nap ( void ) {
 	 * The EFI shell doesn't seem to bother sleeping the CPU; it
 	 * just sits there idly burning power.
 	 *
+	 * If a shutdown is in progess, there may be nothing to
+	 * generate an interrupt since the timer is disabled in the
+	 * first step of ExitBootServices().
 	 */
-	__asm__ __volatile__ ( "wfi" );
+	if ( ! efi_shutdown_in_progress )
+		__asm__ __volatile__ ( "wfi" );
 }
 
 PROVIDE_NAP ( efiarm, cpu_nap, efiarm_cpu_nap );
diff --git a/src/arch/arm32/core/arm32_bigint.c b/src/arch/arm32/core/arm32_bigint.c
index 839bead1..29fb40a7 100644
--- a/src/arch/arm32/core/arm32_bigint.c
+++ b/src/arch/arm32/core/arm32_bigint.c
@@ -36,19 +36,23 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * Multiply big integers
  *
  * @v multiplicand0	Element 0 of big integer to be multiplied
+ * @v multiplicand_size	Number of elements in multiplicand
  * @v multiplier0	Element 0 of big integer to be multiplied
+ * @v multiplier_size	Number of elements in multiplier
  * @v result0		Element 0 of big integer to hold result
- * @v size		Number of elements
  */
 void bigint_multiply_raw ( const uint32_t *multiplicand0,
+			   unsigned int multiplicand_size,
 			   const uint32_t *multiplier0,
-			   uint32_t *result0, unsigned int size ) {
-	const bigint_t ( size ) __attribute__ (( may_alias )) *multiplicand =
-		( ( const void * ) multiplicand0 );
-	const bigint_t ( size ) __attribute__ (( may_alias )) *multiplier =
-		( ( const void * ) multiplier0 );
-	bigint_t ( size * 2 ) __attribute__ (( may_alias )) *result =
-		( ( void * ) result0 );
+			   unsigned int multiplier_size,
+			   uint32_t *result0 ) {
+	unsigned int result_size = ( multiplicand_size + multiplier_size );
+	const bigint_t ( multiplicand_size ) __attribute__ (( may_alias ))
+		*multiplicand = ( ( const void * ) multiplicand0 );
+	const bigint_t ( multiplier_size ) __attribute__ (( may_alias ))
+		*multiplier = ( ( const void * ) multiplier0 );
+	bigint_t ( result_size ) __attribute__ (( may_alias ))
+		*result = ( ( void * ) result0 );
 	unsigned int i;
 	unsigned int j;
 	uint32_t multiplicand_element;
@@ -62,9 +66,9 @@ void bigint_multiply_raw ( const uint32_t *multiplicand0,
 	memset ( result, 0, sizeof ( *result ) );
 
 	/* Multiply integers one element at a time */
-	for ( i = 0 ; i < size ; i++ ) {
+	for ( i = 0 ; i < multiplicand_size ; i++ ) {
 		multiplicand_element = multiplicand->element[i];
-		for ( j = 0 ; j < size ; j++ ) {
+		for ( j = 0 ; j < multiplier_size ; j++ ) {
 			multiplier_element = multiplier->element[j];
 			result_elements = &result->element[ i + j ];
 			/* Perform a single multiply, and add the
@@ -73,7 +77,7 @@ void bigint_multiply_raw ( const uint32_t *multiplicand0,
 			 * never overflow beyond the end of the
 			 * result, since:
 			 *
-			 *     a < 2^{n}, b < 2^{n} => ab < 2^{2n}
+			 *     a < 2^{n}, b < 2^{m} => ab < 2^{n+m}
 			 */
 			__asm__ __volatile__ ( "umull %1, %2, %5, %6\n\t"
 					       "ldr %3, [%0]\n\t"
diff --git a/src/arch/arm32/include/bits/bigint.h b/src/arch/arm32/include/bits/bigint.h
index 103c6c48..e4b511da 100644
--- a/src/arch/arm32/include/bits/bigint.h
+++ b/src/arch/arm32/include/bits/bigint.h
@@ -310,7 +310,9 @@ bigint_done_raw ( const uint32_t *value0, unsigned int size __unused,
 }
 
 extern void bigint_multiply_raw ( const uint32_t *multiplicand0,
+				  unsigned int multiplicand_size,
 				  const uint32_t *multiplier0,
-				  uint32_t *value0, unsigned int size );
+				  unsigned int multiplier_size,
+				  uint32_t *value0 );
 
 #endif /* _BITS_BIGINT_H */
diff --git a/src/arch/arm32/libgcc/lldivmod.S b/src/arch/arm32/libgcc/lldivmod.S
index 746fa8fd..c9c22450 100644
--- a/src/arch/arm32/libgcc/lldivmod.S
+++ b/src/arch/arm32/libgcc/lldivmod.S
@@ -1,7 +1,6 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", %progbits
-	.text
 	.thumb
 
 /**
diff --git a/src/arch/arm32/libgcc/llshift.S b/src/arch/arm32/libgcc/llshift.S
index c1b51e77..592e28e6 100644
--- a/src/arch/arm32/libgcc/llshift.S
+++ b/src/arch/arm32/libgcc/llshift.S
@@ -1,7 +1,6 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", %progbits
-	.text
 	.arm
 
 /**
diff --git a/src/arch/arm64/core/arm64_bigint.c b/src/arch/arm64/core/arm64_bigint.c
index bc4ee9a0..7740f1ae 100644
--- a/src/arch/arm64/core/arm64_bigint.c
+++ b/src/arch/arm64/core/arm64_bigint.c
@@ -36,19 +36,23 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * Multiply big integers
  *
  * @v multiplicand0	Element 0 of big integer to be multiplied
+ * @v multiplicand_size	Number of elements in multiplicand
  * @v multiplier0	Element 0 of big integer to be multiplied
+ * @v multiplier_size	Number of elements in multiplier
  * @v result0		Element 0 of big integer to hold result
- * @v size		Number of elements
  */
 void bigint_multiply_raw ( const uint64_t *multiplicand0,
+			   unsigned int multiplicand_size,
 			   const uint64_t *multiplier0,
-			   uint64_t *result0, unsigned int size ) {
-	const bigint_t ( size ) __attribute__ (( may_alias )) *multiplicand =
-		( ( const void * ) multiplicand0 );
-	const bigint_t ( size ) __attribute__ (( may_alias )) *multiplier =
-		( ( const void * ) multiplier0 );
-	bigint_t ( size * 2 ) __attribute__ (( may_alias )) *result =
-		( ( void * ) result0 );
+			   unsigned int multiplier_size,
+			   uint64_t *result0 ) {
+	unsigned int result_size = ( multiplicand_size + multiplier_size );
+	const bigint_t ( multiplicand_size ) __attribute__ (( may_alias ))
+		*multiplicand = ( ( const void * ) multiplicand0 );
+	const bigint_t ( multiplier_size ) __attribute__ (( may_alias ))
+		*multiplier = ( ( const void * ) multiplier0 );
+	bigint_t ( result_size ) __attribute__ (( may_alias ))
+		*result = ( ( void * ) result0 );
 	unsigned int i;
 	unsigned int j;
 	uint64_t multiplicand_element;
@@ -63,9 +67,9 @@ void bigint_multiply_raw ( const uint64_t *multiplicand0,
 	memset ( result, 0, sizeof ( *result ) );
 
 	/* Multiply integers one element at a time */
-	for ( i = 0 ; i < size ; i++ ) {
+	for ( i = 0 ; i < multiplicand_size ; i++ ) {
 		multiplicand_element = multiplicand->element[i];
-		for ( j = 0 ; j < size ; j++ ) {
+		for ( j = 0 ; j < multiplier_size ; j++ ) {
 			multiplier_element = multiplier->element[j];
 			result_elements = &result->element[ i + j ];
 			/* Perform a single multiply, and add the
@@ -74,7 +78,7 @@ void bigint_multiply_raw ( const uint64_t *multiplicand0,
 			 * never overflow beyond the end of the
 			 * result, since:
 			 *
-			 *     a < 2^{n}, b < 2^{n} => ab < 2^{2n}
+			 *     a < 2^{n}, b < 2^{m} => ab < 2^{n+m}
 			 */
 			__asm__ __volatile__ ( "mul %1, %6, %7\n\t"
 					       "umulh %2, %6, %7\n\t"
diff --git a/src/arch/arm64/include/bits/bigint.h b/src/arch/arm64/include/bits/bigint.h
index 79983b41..0d08bbd6 100644
--- a/src/arch/arm64/include/bits/bigint.h
+++ b/src/arch/arm64/include/bits/bigint.h
@@ -311,7 +311,9 @@ bigint_done_raw ( const uint64_t *value0, unsigned int size __unused,
 }
 
 extern void bigint_multiply_raw ( const uint64_t *multiplicand0,
+				  unsigned int multiplicand_size,
 				  const uint64_t *multiplier0,
-				  uint64_t *value0, unsigned int size );
+				  unsigned int multiplier_size,
+				  uint64_t *value0 );
 
 #endif /* _BITS_BIGINT_H */
diff --git a/src/arch/i386/core/setjmp.S b/src/arch/i386/core/setjmp.S
index e0bbb7ef..cbb5e713 100644
--- a/src/arch/i386/core/setjmp.S
+++ b/src/arch/i386/core/setjmp.S
@@ -2,8 +2,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
 	.text
-	.arch i386
 	.code32
+	.arch i386
 
 	/* Must match jmp_buf structure layout */
 	.struct	0
diff --git a/src/arch/i386/tests/gdbstub_test.S b/src/arch/i386/tests/gdbstub_test.S
index e0c9e6c9..e44c13c2 100644
--- a/src/arch/i386/tests/gdbstub_test.S
+++ b/src/arch/i386/tests/gdbstub_test.S
@@ -1,4 +1,5 @@
 	.section ".note.GNU-stack", "", @progbits
+	.code32
 	.arch i386
 
 	.section ".data", "aw", @progbits
diff --git a/src/arch/loong64/core/loong64_bigint.c b/src/arch/loong64/core/loong64_bigint.c
index f42b8611..b428e22c 100644
--- a/src/arch/loong64/core/loong64_bigint.c
+++ b/src/arch/loong64/core/loong64_bigint.c
@@ -37,19 +37,23 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * Multiply big integers
  *
  * @v multiplicand0	Element 0 of big integer to be multiplied
+ * @v multiplicand_size	Number of elements in multiplicand
  * @v multiplier0	Element 0 of big integer to be multiplied
+ * @v multiplier_size	Number of elements in multiplier
  * @v result0		Element 0 of big integer to hold result
- * @v size		Number of elements
  */
 void bigint_multiply_raw ( const uint64_t *multiplicand0,
+			   unsigned int multiplicand_size,
 			   const uint64_t *multiplier0,
-			   uint64_t *result0, unsigned int size ) {
-	const bigint_t ( size ) __attribute__ (( may_alias )) *multiplicand =
-		( ( const void * ) multiplicand0 );
-	const bigint_t ( size ) __attribute__ (( may_alias )) *multiplier =
-		( ( const void * ) multiplier0 );
-	bigint_t ( size * 2 ) __attribute__ (( may_alias )) *result =
-		( ( void * ) result0 );
+			   unsigned int multiplier_size,
+			   uint64_t *result0 ) {
+	unsigned int result_size = ( multiplicand_size + multiplier_size );
+	const bigint_t ( multiplicand_size ) __attribute__ (( may_alias ))
+		*multiplicand = ( ( const void * ) multiplicand0 );
+	const bigint_t ( multiplier_size ) __attribute__ (( may_alias ))
+		*multiplier = ( ( const void * ) multiplier0 );
+	bigint_t ( result_size ) __attribute__ (( may_alias ))
+		*result = ( ( void * ) result0 );
 	unsigned int i;
 	unsigned int j;
 	uint64_t multiplicand_element;
@@ -64,9 +68,9 @@ void bigint_multiply_raw ( const uint64_t *multiplicand0,
 	memset ( result, 0, sizeof ( *result ) );
 
 	/* Multiply integers one element at a time */
-	for ( i = 0 ; i < size ; i++ ) {
+	for ( i = 0 ; i < multiplicand_size ; i++ ) {
 		multiplicand_element = multiplicand->element[i];
-		for ( j = 0 ; j < size ; j++ ) {
+		for ( j = 0 ; j < multiplier_size ; j++ ) {
 			multiplier_element = multiplier->element[j];
 			result_elements = &result->element[ i + j ];
 			/* Perform a single multiply, and add the
@@ -75,7 +79,7 @@ void bigint_multiply_raw ( const uint64_t *multiplicand0,
 			 * never overflow beyond the end of the
 			 * result, since:
 			 *
-			 *     a < 2^{n}, b < 2^{n} => ab < 2^{2n}
+			 *     a < 2^{n}, b < 2^{m} => ab < 2^{n+m}
 			 */
 			__asm__ __volatile__ ( "mul.d   %1, %6, %7\n\t"
 					       "mulh.du %2, %6, %7\n\t"
diff --git a/src/arch/loong64/include/bits/bigint.h b/src/arch/loong64/include/bits/bigint.h
index 89e0b867..bec748be 100644
--- a/src/arch/loong64/include/bits/bigint.h
+++ b/src/arch/loong64/include/bits/bigint.h
@@ -53,34 +53,37 @@ bigint_add_raw ( const uint64_t *addend0, uint64_t *value0,
 	uint64_t *discard_value;
 	uint64_t discard_addend_i;
 	uint64_t discard_value_i;
+	uint64_t discard_carry;
+	uint64_t discard_temp;
 	unsigned int discard_size;
-	__asm__ __volatile__ ( "move   $t0, $zero\n"
-			       "1:\n\t"
-			       "ld.d   %3, %0, 0\n\t"
-			       "addi.d %0, %0, 8\n\t"
-			       "ld.d   %4, %1, 0\n\t"
-
-			       "add.d  %4, %4, $t0\n\t"
-			       "sltu   $t0, %4, $t0\n\t"
-
-			       "add.d  %4, %4, %3\n\t"
-			       "sltu   $t1, %4, %3\n\t"
 
-			       "or     $t0, $t0, $t1\n\t"
-			       "st.d   %4,  %1, 0\n\t"
+	__asm__ __volatile__ ( "\n1:\n\t"
+			       /* Load addend[i] and value[i] */
+			       "ld.d %3, %0, 0\n\t"
+			       "ld.d %4, %1, 0\n\t"
+			       /* Add carry flag and addend */
+			       "add.d %4, %4, %5\n\t"
+			       "sltu %6, %4, %5\n\t"
+			       "add.d %4, %4, %3\n\t"
+			       "sltu %5, %4, %3\n\t"
+			       "or %5, %5, %6\n\t"
+			       /* Store value[i] */
+			       "st.d %4, %1, 0\n\t"
+			       /* Loop */
+			       "addi.d %0, %0, 8\n\t"
 			       "addi.d %1, %1, 8\n\t"
 			       "addi.w %2, %2, -1\n\t"
-			       "bnez   %2, 1b"
+			       "bnez %2, 1b\n\t"
 			       : "=r" ( discard_addend ),
 				 "=r" ( discard_value ),
 				 "=r" ( discard_size ),
 				 "=r" ( discard_addend_i ),
 				 "=r" ( discard_value_i ),
+				 "=r" ( discard_carry ),
+				 "=r" ( discard_temp ),
 				 "+m" ( *value )
-			       : "0" ( addend0 ),
-				 "1" ( value0 ),
-				 "2" ( size )
-			       : "t0", "t1" );
+			       : "0" ( addend0 ), "1" ( value0 ),
+				 "2" ( size ), "5" ( 0 ) );
 }
 
 /**
@@ -93,35 +96,43 @@ bigint_add_raw ( const uint64_t *addend0, uint64_t *value0,
 static inline __attribute__ (( always_inline )) void
 bigint_subtract_raw ( const uint64_t *subtrahend0, uint64_t *value0,
 		      unsigned int size ) {
+	bigint_t ( size ) __attribute__ (( may_alias )) *value =
+		( ( void * ) value0 );
 	uint64_t *discard_subtrahend;
 	uint64_t *discard_value;
 	uint64_t discard_subtrahend_i;
 	uint64_t discard_value_i;
+	uint64_t discard_carry;
+	uint64_t discard_temp;
 	unsigned int discard_size;
-	unsigned int flag = 0;
-
-	discard_subtrahend = (uint64_t*) subtrahend0;
-	discard_value = value0;
-	discard_size = size;
-
-	do {
-		discard_subtrahend_i = *discard_subtrahend;
-		discard_subtrahend++;
-		discard_value_i = *discard_value;
-
-		discard_value_i = discard_value_i - discard_subtrahend_i - flag;
-
-		if ( *discard_value < (discard_subtrahend_i + flag)) {
-			flag = 1;
-		} else {
-			flag = 0;
-		}
 
-		*discard_value = discard_value_i;
-
-		discard_value++;
-		discard_size -= 1;
-	} while (discard_size != 0);
+	__asm__ __volatile__ ( "\n1:\n\t"
+			       /* Load subtrahend[i] and value[i] */
+			       "ld.d %3, %0, 0\n\t"
+			       "ld.d %4, %1, 0\n\t"
+			       /* Subtract carry flag and subtrahend */
+			       "sltu %6, %4, %5\n\t"
+			       "sub.d %4, %4, %5\n\t"
+			       "sltu %5, %4, %3\n\t"
+			       "sub.d %4, %4, %3\n\t"
+			       "or %5, %5, %6\n\t"
+			       /* Store value[i] */
+			       "st.d %4, %1, 0\n\t"
+			       /* Loop */
+			       "addi.d %0, %0, 8\n\t"
+			       "addi.d %1, %1, 8\n\t"
+			       "addi.w %2, %2, -1\n\t"
+			       "bnez %2, 1b\n\t"
+			       : "=r" ( discard_subtrahend ),
+				 "=r" ( discard_value ),
+				 "=r" ( discard_size ),
+				 "=r" ( discard_subtrahend_i ),
+				 "=r" ( discard_value_i ),
+				 "=r" ( discard_carry ),
+				 "=r" ( discard_temp ),
+				 "+m" ( *value )
+			       : "0" ( subtrahend0 ), "1" ( value0 ),
+				 "2" ( size ), "5" ( 0 ) );
 }
 
 /**
@@ -132,30 +143,37 @@ bigint_subtract_raw ( const uint64_t *subtrahend0, uint64_t *value0,
  */
 static inline __attribute__ (( always_inline )) void
 bigint_rol_raw ( uint64_t *value0, unsigned int size ) {
+	bigint_t ( size ) __attribute__ (( may_alias )) *value =
+		( ( void * ) value0 );
 	uint64_t *discard_value;
 	uint64_t discard_value_i;
+	uint64_t discard_carry;
+	uint64_t discard_temp;
 	unsigned int discard_size;
-	uint64_t current_value_i;
-	unsigned int flag = 0;
 
-	discard_value = value0;
-	discard_size = size;
-	do {
-		discard_value_i = *discard_value;
-		current_value_i = discard_value_i;
-
-		discard_value_i += discard_value_i + flag;
-
-		if (discard_value_i < current_value_i) {
-			flag = 1;
-		} else {
-			flag = 0;
-		}
-
-		*discard_value = discard_value_i;
-		discard_value++;
-		discard_size -= 1;
-	} while ( discard_size != 0 );
+	__asm__ __volatile__ ( "\n1:\n\t"
+			       /* Load value[i] */
+			       "ld.d %2, %0, 0\n\t"
+			       /* Shift left */
+			       "rotri.d %2, %2, 63\n\t"
+			       "andi %4, %2, 1\n\t"
+			       "xor %2, %2, %4\n\t"
+			       "or %2, %2, %3\n\t"
+			       "move %3, %4\n\t"
+			       /* Store value[i] */
+			       "st.d %2, %0, 0\n\t"
+			       /* Loop  */
+			       "addi.d %0, %0, 8\n\t"
+			       "addi.w %1, %1, -1\n\t"
+			       "bnez %1, 1b\n\t"
+			       : "=r" ( discard_value ),
+				 "=r" ( discard_size ),
+				 "=r" ( discard_value_i ),
+				 "=r" ( discard_carry ),
+				 "=r" ( discard_temp ),
+				 "+m" ( *value )
+			       : "0" ( value0 ), "1" ( size ), "3" ( 0 )
+			       : "cc" );
 }
 
 /**
@@ -166,27 +184,37 @@ bigint_rol_raw ( uint64_t *value0, unsigned int size ) {
  */
 static inline __attribute__ (( always_inline )) void
 bigint_ror_raw ( uint64_t *value0, unsigned int size ) {
+	bigint_t ( size ) __attribute__ (( may_alias )) *value =
+		( ( void * ) value0 );
 	uint64_t *discard_value;
 	uint64_t discard_value_i;
-	uint64_t discard_value_j;
+	uint64_t discard_carry;
+	uint64_t discard_temp;
 	unsigned int discard_size;
 
-	discard_value = value0;
-	discard_size = size;
-
-	discard_value_j = 0;
-
-	do {
-		discard_size -= 1;
-
-		discard_value_i = *(discard_value + discard_size);
-
-		discard_value_j = (discard_value_j << 63) | (discard_value_i >> 1);
-
-		*(discard_value + discard_size) = discard_value_j;
-
-		discard_value_j = discard_value_i;
-	} while ( discard_size > 0 );
+	__asm__ __volatile__ ( "\n1:\n\t"
+			       /* Load value[i] */
+			       "ld.d %2, %0, -8\n\t"
+			       /* Shift right */
+			       "andi %4, %2, 1\n\t"
+			       "xor %2, %2, %4\n\t"
+			       "or %2, %2, %3\n\t"
+			       "move %3, %4\n\t"
+			       "rotri.d %2, %2, 1\n\t"
+			       /* Store value[i] */
+			       "st.d %2, %0, -8\n\t"
+			       /* Loop  */
+			       "addi.d %0, %0, -8\n\t"
+			       "addi.w %1, %1, -1\n\t"
+			       "bnez %1, 1b\n\t"
+			       : "=r" ( discard_value ),
+				 "=r" ( discard_size ),
+				 "=r" ( discard_value_i ),
+				 "=r" ( discard_carry ),
+				 "=r" ( discard_temp ),
+				 "+m" ( *value )
+			       : "0" ( value0 + size ), "1" ( size ), "3" ( 0 )
+			       : "cc" );
 }
 
 /**
@@ -330,7 +358,9 @@ bigint_done_raw ( const uint64_t *value0, unsigned int size __unused,
 }
 
 extern void bigint_multiply_raw ( const uint64_t *multiplicand0,
+				  unsigned int multiplicand_size,
 				  const uint64_t *multiplier0,
-				  uint64_t *value0, unsigned int size );
+				  unsigned int multiplier_size,
+				  uint64_t *value0 );
 
 #endif /* _BITS_BIGINT_H */
diff --git a/src/arch/loong64/interface/efi/efiloong64_nap.c b/src/arch/loong64/interface/efi/efiloong64_nap.c
index 5cd1c1b9..0a760978 100644
--- a/src/arch/loong64/interface/efi/efiloong64_nap.c
+++ b/src/arch/loong64/interface/efi/efiloong64_nap.c
@@ -46,8 +46,12 @@ static void efiloong64_cpu_nap ( void ) {
 	 * The EFI shell doesn't seem to bother sleeping the CPU; it
 	 * just sits there idly burning power.
 	 *
+	 * If a shutdown is in progess, there may be nothing to
+	 * generate an interrupt since the timer is disabled in the
+	 * first step of ExitBootServices().
 	 */
-	__asm__ __volatile__ ( "idle 0" );
+	if ( ! efi_shutdown_in_progress )
+		__asm__ __volatile__ ( "idle 0" );
 }
 
 PROVIDE_NAP ( efiloong64, cpu_nap, efiloong64_cpu_nap );
diff --git a/src/arch/x86/core/patch_cf.S b/src/arch/x86/core/patch_cf.S
index 63730c3f..62f19e45 100644
--- a/src/arch/x86/core/patch_cf.S
+++ b/src/arch/x86/core/patch_cf.S
@@ -23,9 +23,8 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
-	.arch i386
 	.code16
+	.arch i386
 
 /****************************************************************************
  * Set/clear CF on the stack as appropriate, assumes stack is as it should
diff --git a/src/arch/x86/core/stack.S b/src/arch/x86/core/stack.S
index 49345347..1bcaf18f 100644
--- a/src/arch/x86/core/stack.S
+++ b/src/arch/x86/core/stack.S
@@ -1,7 +1,6 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
-	.arch i386
 
 #ifdef __x86_64__
 #define STACK_SIZE 8192
diff --git a/src/arch/x86/core/stack16.S b/src/arch/x86/core/stack16.S
index d3949a55..622887ea 100644
--- a/src/arch/x86/core/stack16.S
+++ b/src/arch/x86/core/stack16.S
@@ -1,7 +1,6 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
-	.arch i386
 
 /****************************************************************************
  * Internal stack
diff --git a/src/arch/x86/core/x86_bigint.c b/src/arch/x86/core/x86_bigint.c
index 9a25bdad..74e5da9a 100644
--- a/src/arch/x86/core/x86_bigint.c
+++ b/src/arch/x86/core/x86_bigint.c
@@ -36,19 +36,23 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * Multiply big integers
  *
  * @v multiplicand0	Element 0 of big integer to be multiplied
+ * @v multiplicand_size	Number of elements in multiplicand
  * @v multiplier0	Element 0 of big integer to be multiplied
+ * @v multiplier_size	Number of elements in multiplier
  * @v result0		Element 0 of big integer to hold result
- * @v size		Number of elements
  */
 void bigint_multiply_raw ( const uint32_t *multiplicand0,
+			   unsigned int multiplicand_size,
 			   const uint32_t *multiplier0,
-			   uint32_t *result0, unsigned int size ) {
-	const bigint_t ( size ) __attribute__ (( may_alias )) *multiplicand =
-		( ( const void * ) multiplicand0 );
-	const bigint_t ( size ) __attribute__ (( may_alias )) *multiplier =
-		( ( const void * ) multiplier0 );
-	bigint_t ( size * 2 ) __attribute__ (( may_alias )) *result =
-		( ( void * ) result0 );
+			   unsigned int multiplier_size,
+			   uint32_t *result0 ) {
+	unsigned int result_size = ( multiplicand_size + multiplier_size );
+	const bigint_t ( multiplicand_size ) __attribute__ (( may_alias ))
+		*multiplicand = ( ( const void * ) multiplicand0 );
+	const bigint_t ( multiplier_size ) __attribute__ (( may_alias ))
+		*multiplier = ( ( const void * ) multiplier0 );
+	bigint_t ( result_size ) __attribute__ (( may_alias ))
+		*result = ( ( void * ) result0 );
 	unsigned int i;
 	unsigned int j;
 	uint32_t multiplicand_element;
@@ -62,9 +66,9 @@ void bigint_multiply_raw ( const uint32_t *multiplicand0,
 	memset ( result, 0, sizeof ( *result ) );
 
 	/* Multiply integers one element at a time */
-	for ( i = 0 ; i < size ; i++ ) {
+	for ( i = 0 ; i < multiplicand_size ; i++ ) {
 		multiplicand_element = multiplicand->element[i];
-		for ( j = 0 ; j < size ; j++ ) {
+		for ( j = 0 ; j < multiplier_size ; j++ ) {
 			multiplier_element = multiplier->element[j];
 			result_elements = &result->element[ i + j ];
 			/* Perform a single multiply, and add the
@@ -73,7 +77,7 @@ void bigint_multiply_raw ( const uint32_t *multiplicand0,
 			 * never overflow beyond the end of the
 			 * result, since:
 			 *
-			 *     a < 2^{n}, b < 2^{n} => ab < 2^{2n}
+			 *     a < 2^{n}, b < 2^{m} => ab < 2^{n+m}
 			 */
 			__asm__ __volatile__ ( "mull %5\n\t"
 					       "addl %%eax, (%6,%2,4)\n\t"
diff --git a/src/arch/x86/drivers/net/undiisr.S b/src/arch/x86/drivers/net/undiisr.S
index a1098b83..8ba5c535 100644
--- a/src/arch/x86/drivers/net/undiisr.S
+++ b/src/arch/x86/drivers/net/undiisr.S
@@ -11,9 +11,8 @@ FILE_LICENCE ( GPL2_OR_LATER )
 #define PIC2_ICR 0xa0
 	
 	.section ".note.GNU-stack", "", @progbits
-	.text
-	.arch i386
 	.code16
+	.arch i386
 
 	.section ".text16", "ax", @progbits
 	.globl undiisr
diff --git a/src/arch/x86/include/bits/bigint.h b/src/arch/x86/include/bits/bigint.h
index 7443d6fd..a6bc2ca1 100644
--- a/src/arch/x86/include/bits/bigint.h
+++ b/src/arch/x86/include/bits/bigint.h
@@ -323,7 +323,9 @@ bigint_done_raw ( const uint32_t *value0, unsigned int size __unused,
 }
 
 extern void bigint_multiply_raw ( const uint32_t *multiplicand0,
+				  unsigned int multiplicand_size,
 				  const uint32_t *multiplier0,
-				  uint32_t *value0, unsigned int size );
+				  unsigned int multiplier_size,
+				  uint32_t *value0 );
 
 #endif /* _BITS_BIGINT_H */
diff --git a/src/arch/x86/interface/efi/efix86_nap.c b/src/arch/x86/interface/efi/efix86_nap.c
index 3ebf0bd6..296876b8 100644
--- a/src/arch/x86/interface/efi/efix86_nap.c
+++ b/src/arch/x86/interface/efi/efix86_nap.c
@@ -46,8 +46,12 @@ static void efix86_cpu_nap ( void ) {
 	 * The EFI shell doesn't seem to bother sleeping the CPU; it
 	 * just sits there idly burning power.
 	 *
+	 * If a shutdown is in progess, there may be nothing to
+	 * generate an interrupt since the timer is disabled in the
+	 * first step of ExitBootServices().
 	 */
-	__asm__ __volatile__ ( "hlt" );
+	if ( ! efi_shutdown_in_progress )
+		__asm__ __volatile__ ( "hlt" );
 }
 
 PROVIDE_NAP ( efix86, cpu_nap, efix86_cpu_nap );
diff --git a/src/arch/x86/interface/pcbios/bios_smbios.c b/src/arch/x86/interface/pcbios/bios_smbios.c
index a8c0fc32..366679d3 100644
--- a/src/arch/x86/interface/pcbios/bios_smbios.c
+++ b/src/arch/x86/interface/pcbios/bios_smbios.c
@@ -44,11 +44,11 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * @v smbios		SMBIOS entry point descriptor structure to fill in
  * @ret rc		Return status code
  */
-static int bios_find_smbios ( struct smbios *smbios ) {
+static int bios_find_smbios2 ( struct smbios *smbios ) {
 	struct smbios_entry entry;
 	int rc;
 
-	/* Scan through BIOS segment to find SMBIOS entry point */
+	/* Scan through BIOS segment to find SMBIOS 32-bit entry point */
 	if ( ( rc = find_smbios_entry ( real_to_user ( BIOS_SEG, 0 ), 0x10000,
 					&entry ) ) != 0 )
 		return rc;
@@ -62,4 +62,55 @@ static int bios_find_smbios ( struct smbios *smbios ) {
 	return 0;
 }
 
+/**
+ * Find SMBIOS
+ *
+ * @v smbios		SMBIOS entry point descriptor structure to fill in
+ * @ret rc		Return status code
+ */
+static int bios_find_smbios3 ( struct smbios *smbios ) {
+	struct smbios3_entry entry;
+	int rc;
+
+	/* Scan through BIOS segment to find SMBIOS 64-bit entry point */
+	if ( ( rc = find_smbios3_entry ( real_to_user ( BIOS_SEG, 0 ), 0x10000,
+					 &entry ) ) != 0 )
+		return rc;
+
+	/* Check that address is accessible */
+	if ( entry.smbios_address > ~( ( physaddr_t ) 0 ) ) {
+		DBG ( "SMBIOS3 at %08llx is inaccessible\n",
+		      ( ( unsigned long long ) entry.smbios_address ) );
+		return -ENOTSUP;
+	}
+
+	/* Fill in entry point descriptor structure */
+	smbios->address = phys_to_user ( entry.smbios_address );
+	smbios->len = entry.smbios_len;
+	smbios->count = 0;
+	smbios->version = SMBIOS_VERSION ( entry.major, entry.minor );
+
+	return 0;
+}
+
+/**
+ * Find SMBIOS
+ *
+ * @v smbios		SMBIOS entry point descriptor structure to fill in
+ * @ret rc		Return status code
+ */
+static int bios_find_smbios ( struct smbios *smbios ) {
+	int rc;
+
+	/* Use 32-bit table if present */
+	if ( ( rc = bios_find_smbios2 ( smbios ) ) == 0 )
+		return 0;
+
+	/* Otherwise, use 64-bit table if present and accessible */
+	if ( ( rc = bios_find_smbios3 ( smbios ) ) == 0 )
+		return 0;
+
+	return rc;
+}
+
 PROVIDE_SMBIOS ( pcbios, find_smbios, bios_find_smbios );
diff --git a/src/arch/x86/interface/pcbios/e820mangler.S b/src/arch/x86/interface/pcbios/e820mangler.S
index 46e1cab4..ef5dc275 100644
--- a/src/arch/x86/interface/pcbios/e820mangler.S
+++ b/src/arch/x86/interface/pcbios/e820mangler.S
@@ -24,9 +24,8 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
-	.arch i386
 	.code16
+	.arch i386
 
 #define SMAP 0x534d4150
 
diff --git a/src/arch/x86/interface/pcbios/int13.c b/src/arch/x86/interface/pcbios/int13.c
index d6c4d7eb..372d40ba 100644
--- a/src/arch/x86/interface/pcbios/int13.c
+++ b/src/arch/x86/interface/pcbios/int13.c
@@ -183,8 +183,8 @@ static int int13_parse_eltorito ( struct san_device *sandev, void *scratch ) {
 	/* Read boot record volume descriptor */
 	if ( ( rc = sandev_read ( sandev, ELTORITO_LBA, 1,
 				  virt_to_user ( boot ) ) ) != 0 ) {
-		DBGC ( sandev, "INT13 drive %02x could not read El Torito boot "
-		       "record volume descriptor: %s\n",
+		DBGC ( sandev->drive, "INT13 drive %02x could not read El "
+		       "Torito boot record volume descriptor: %s\n",
 		       sandev->drive, strerror ( rc ) );
 		return rc;
 	}
@@ -192,10 +192,11 @@ static int int13_parse_eltorito ( struct san_device *sandev, void *scratch ) {
 	/* Check for an El Torito boot catalog */
 	if ( memcmp ( boot, &boot_check, sizeof ( boot_check ) ) == 0 ) {
 		int13->boot_catalog = boot->sector;
-		DBGC ( sandev, "INT13 drive %02x has an El Torito boot catalog "
-		       "at LBA %08x\n", sandev->drive, int13->boot_catalog );
+		DBGC ( sandev->drive, "INT13 drive %02x has an El Torito boot "
+		       "catalog at LBA %08x\n", sandev->drive,
+		       int13->boot_catalog );
 	} else {
-		DBGC ( sandev, "INT13 drive %02x has no El Torito boot "
+		DBGC ( sandev->drive, "INT13 drive %02x has no El Torito boot "
 		       "catalog\n", sandev->drive );
 	}
 
@@ -228,14 +229,14 @@ static int int13_guess_geometry_hdd ( struct san_device *sandev, void *scratch,
 
 	/* Read partition table */
 	if ( ( rc = sandev_read ( sandev, 0, 1, virt_to_user ( mbr ) ) ) != 0 ) {
-		DBGC ( sandev, "INT13 drive %02x could not read "
+		DBGC ( sandev->drive, "INT13 drive %02x could not read "
 		       "partition table to guess geometry: %s\n",
 		       sandev->drive, strerror ( rc ) );
 		return rc;
 	}
-	DBGC2 ( sandev, "INT13 drive %02x has MBR:\n", sandev->drive );
-	DBGC2_HDA ( sandev, 0, mbr, sizeof ( *mbr ) );
-	DBGC ( sandev, "INT13 drive %02x has signature %08x\n",
+	DBGC2 ( sandev->drive, "INT13 drive %02x has MBR:\n", sandev->drive );
+	DBGC2_HDA ( sandev->drive, 0, mbr, sizeof ( *mbr ) );
+	DBGC ( sandev->drive, "INT13 drive %02x has signature %08x\n",
 	       sandev->drive, mbr->signature );
 
 	/* Scan through partition table and modify guesses for
@@ -260,8 +261,8 @@ static int int13_guess_geometry_hdd ( struct san_device *sandev, void *scratch,
 		if ( ( start_cylinder == 0 ) && ( start_head != 0 ) ) {
 			*sectors = ( ( partition->start + 1 - start_sector ) /
 				     start_head );
-			DBGC ( sandev, "INT13 drive %02x guessing C/H/S "
-			       "xx/xx/%d based on partition %d\n",
+			DBGC ( sandev->drive, "INT13 drive %02x guessing "
+			       "C/H/S xx/xx/%d based on partition %d\n",
 			       sandev->drive, *sectors, ( i + 1 ) );
 		}
 
@@ -272,14 +273,14 @@ static int int13_guess_geometry_hdd ( struct san_device *sandev, void *scratch,
 		end_sector = PART_SECTOR ( partition->chs_end );
 		if ( ( end_head + 1 ) > *heads ) {
 			*heads = ( end_head + 1 );
-			DBGC ( sandev, "INT13 drive %02x guessing C/H/S "
-			       "xx/%d/xx based on partition %d\n",
+			DBGC ( sandev->drive, "INT13 drive %02x guessing "
+			       "C/H/S xx/%d/xx based on partition %d\n",
 			       sandev->drive, *heads, ( i + 1 ) );
 		}
 		if ( end_sector > *sectors ) {
 			*sectors = end_sector;
-			DBGC ( sandev, "INT13 drive %02x guessing C/H/S "
-			       "xx/xx/%d based on partition %d\n",
+			DBGC ( sandev->drive, "INT13 drive %02x guessing "
+			       "C/H/S xx/xx/%d based on partition %d\n",
 			       sandev->drive, *sectors, ( i + 1 ) );
 		}
 	}
@@ -343,9 +344,10 @@ static int int13_guess_geometry_fdd ( struct san_device *sandev,
 		*heads = INT13_FDD_HEADS ( geometry );
 		*sectors = INT13_FDD_SECTORS ( geometry );
 		if ( ( cylinders * (*heads) * (*sectors) ) == blocks ) {
-			DBGC ( sandev, "INT13 drive %02x guessing C/H/S "
-			       "%d/%d/%d based on size %dK\n", sandev->drive,
-			       cylinders, *heads, *sectors, ( blocks / 2 ) );
+			DBGC ( sandev->drive, "INT13 drive %02x guessing "
+			       "C/H/S %d/%d/%d based on size %dK\n",
+			       sandev->drive, cylinders, *heads, *sectors,
+			       ( blocks / 2 ) );
 			return 0;
 		}
 	}
@@ -355,8 +357,9 @@ static int int13_guess_geometry_fdd ( struct san_device *sandev,
 	 */
 	*heads = 2;
 	*sectors = 18;
-	DBGC ( sandev, "INT13 drive %02x guessing C/H/S xx/%d/%d based on size "
-	       "%dK\n", sandev->drive, *heads, *sectors, ( blocks / 2 ) );
+	DBGC ( sandev->drive, "INT13 drive %02x guessing C/H/S xx/%d/%d "
+	       "based on size %dK\n", sandev->drive, *heads, *sectors,
+	       ( blocks / 2 ) );
 	return 0;
 }
 
@@ -431,8 +434,8 @@ static void int13_sync_num_drives ( void ) {
 		required = ( ( max_drive & 0x7f ) + 1 );
 		if ( *counter < required ) {
 			*counter = required;
-			DBGC ( sandev, "INT13 drive %02x added to drive count: "
-			       "%d HDDs, %d FDDs\n",
+			DBGC ( sandev->drive, "INT13 drive %02x added to "
+			       "drive count: %d HDDs, %d FDDs\n",
 			       sandev->drive, num_drives, num_fdds );
 		}
 	}
@@ -472,7 +475,7 @@ static int int13_reset ( struct san_device *sandev,
 			 struct i386_all_regs *ix86 __unused ) {
 	int rc;
 
-	DBGC2 ( sandev, "Reset drive\n" );
+	DBGC2 ( sandev->drive, "Reset drive\n" );
 
 	/* Reset SAN device */
 	if ( ( rc = sandev_reset ( sandev ) ) != 0 )
@@ -491,7 +494,7 @@ static int int13_get_last_status ( struct san_device *sandev,
 				   struct i386_all_regs *ix86 __unused ) {
 	struct int13_data *int13 = sandev->priv;
 
-	DBGC2 ( sandev, "Get status of last operation\n" );
+	DBGC2 ( sandev->drive, "Get status of last operation\n" );
 	return int13->last_status;
 }
 
@@ -524,8 +527,8 @@ static int int13_rw_sectors ( struct san_device *sandev,
 
 	/* Validate blocksize */
 	if ( sandev_blksize ( sandev ) != INT13_BLKSIZE ) {
-		DBGC ( sandev, "\nINT 13 drive %02x invalid blocksize (%zd) "
-		       "for non-extended read/write\n",
+		DBGC ( sandev->drive, "\nINT 13 drive %02x invalid blocksize "
+		       "(%zd) for non-extended read/write\n",
 		       sandev->drive, sandev_blksize ( sandev ) );
 		return -INT13_STATUS_INVALID;
 	}
@@ -537,9 +540,10 @@ static int int13_rw_sectors ( struct san_device *sandev,
 	if ( ( cylinder >= int13->cylinders ) ||
 	     ( head >= int13->heads ) ||
 	     ( sector < 1 ) || ( sector > int13->sectors_per_track ) ) {
-		DBGC ( sandev, "C/H/S %d/%d/%d out of range for geometry "
-		       "%d/%d/%d\n", cylinder, head, sector, int13->cylinders,
-		       int13->heads, int13->sectors_per_track );
+		DBGC ( sandev->drive, "C/H/S %d/%d/%d out of range for "
+		       "geometry %d/%d/%d\n", cylinder, head, sector,
+		       int13->cylinders, int13->heads,
+		       int13->sectors_per_track );
 		return -INT13_STATUS_INVALID;
 	}
 	lba = ( ( ( ( cylinder * int13->heads ) + head )
@@ -547,13 +551,13 @@ static int int13_rw_sectors ( struct san_device *sandev,
 	count = ix86->regs.al;
 	buffer = real_to_user ( ix86->segs.es, ix86->regs.bx );
 
-	DBGC2 ( sandev, "C/H/S %d/%d/%d = LBA %08lx <-> %04x:%04x (count %d)\n",
-		cylinder, head, sector, lba, ix86->segs.es, ix86->regs.bx,
-		count );
+	DBGC2 ( sandev->drive, "C/H/S %d/%d/%d = LBA %08lx <-> %04x:%04x "
+		"(count %d)\n", cylinder, head, sector, lba, ix86->segs.es,
+		ix86->regs.bx, count );
 
 	/* Read from / write to block device */
 	if ( ( rc = sandev_rw ( sandev, lba, count, buffer ) ) != 0 ){
-		DBGC ( sandev, "INT13 drive %02x I/O failed: %s\n",
+		DBGC ( sandev->drive, "INT13 drive %02x I/O failed: %s\n",
 		       sandev->drive, strerror ( rc ) );
 		return -INT13_STATUS_READ_ERROR;
 	}
@@ -577,7 +581,7 @@ static int int13_rw_sectors ( struct san_device *sandev,
 static int int13_read_sectors ( struct san_device *sandev,
 				struct i386_all_regs *ix86 ) {
 
-	DBGC2 ( sandev, "Read: " );
+	DBGC2 ( sandev->drive, "Read: " );
 	return int13_rw_sectors ( sandev, ix86, sandev_read );
 }
 
@@ -597,7 +601,7 @@ static int int13_read_sectors ( struct san_device *sandev,
 static int int13_write_sectors ( struct san_device *sandev,
 				 struct i386_all_regs *ix86 ) {
 
-	DBGC2 ( sandev, "Write: " );
+	DBGC2 ( sandev->drive, "Write: " );
 	return int13_rw_sectors ( sandev, ix86, sandev_write );
 }
 
@@ -619,12 +623,12 @@ static int int13_get_parameters ( struct san_device *sandev,
 	unsigned int max_head = int13->heads - 1;
 	unsigned int max_sector = int13->sectors_per_track; /* sic */
 
-	DBGC2 ( sandev, "Get drive parameters\n" );
+	DBGC2 ( sandev->drive, "Get drive parameters\n" );
 
 	/* Validate blocksize */
 	if ( sandev_blksize ( sandev ) != INT13_BLKSIZE ) {
-		DBGC ( sandev, "\nINT 13 drive %02x invalid blocksize (%zd) "
-		       "for non-extended parameters\n",
+		DBGC ( sandev->drive, "\nINT 13 drive %02x invalid blocksize "
+		       "(%zd) for non-extended parameters\n",
 		       sandev->drive, sandev_blksize ( sandev ) );
 		return -INT13_STATUS_INVALID;
 	}
@@ -657,7 +661,7 @@ static int int13_get_disk_type ( struct san_device *sandev,
 				 struct i386_all_regs *ix86 ) {
 	uint32_t blocks;
 
-	DBGC2 ( sandev, "Get disk type\n" );
+	DBGC2 ( sandev->drive, "Get disk type\n" );
 
 	if ( int13_is_fdd ( sandev ) ) {
 		return INT13_DISK_TYPE_FDD;
@@ -682,7 +686,7 @@ static int int13_extension_check ( struct san_device *sandev,
 				   struct i386_all_regs *ix86 ) {
 
 	if ( ( ix86->regs.bx == 0x55aa ) && ! int13_is_fdd ( sandev ) ) {
-		DBGC2 ( sandev, "INT13 extensions installation check\n" );
+		DBGC2 ( sandev->drive, "INT13 extensions check\n" );
 		ix86->regs.bx = 0xaa55;
 		ix86->regs.cx = ( INT13_EXTENSION_LINEAR |
 				  INT13_EXTENSION_EDD |
@@ -725,7 +729,8 @@ static int int13_extended_rw ( struct san_device *sandev,
 	get_real ( bufsize, ix86->segs.ds,
 		   ( ix86->regs.si + offsetof ( typeof ( addr ), bufsize ) ) );
 	if ( bufsize < offsetof ( typeof ( addr ), buffer_phys ) ) {
-		DBGC2 ( sandev, "<invalid buffer size %#02x\n>\n", bufsize );
+		DBGC2 ( sandev->drive, "<invalid buffer size %#02x\n>\n",
+			bufsize );
 		return -INT13_STATUS_INVALID;
 	}
 
@@ -733,17 +738,18 @@ static int int13_extended_rw ( struct san_device *sandev,
 	memset ( &addr, 0, sizeof ( addr ) );
 	copy_from_real ( &addr, ix86->segs.ds, ix86->regs.si, bufsize );
 	lba = addr.lba;
-	DBGC2 ( sandev, "LBA %08llx <-> ", ( ( unsigned long long ) lba ) );
+	DBGC2 ( sandev->drive, "LBA %08llx <-> ",
+		( ( unsigned long long ) lba ) );
 	if ( ( addr.count == 0xff ) ||
 	     ( ( addr.buffer.segment == 0xffff ) &&
 	       ( addr.buffer.offset == 0xffff ) ) ) {
 		buffer = phys_to_user ( addr.buffer_phys );
-		DBGC2 ( sandev, "%08llx",
+		DBGC2 ( sandev->drive, "%08llx",
 			( ( unsigned long long ) addr.buffer_phys ) );
 	} else {
 		buffer = real_to_user ( addr.buffer.segment,
 					addr.buffer.offset );
-		DBGC2 ( sandev, "%04x:%04x", addr.buffer.segment,
+		DBGC2 ( sandev->drive, "%04x:%04x", addr.buffer.segment,
 			addr.buffer.offset );
 	}
 	if ( addr.count <= 0x7f ) {
@@ -751,15 +757,15 @@ static int int13_extended_rw ( struct san_device *sandev,
 	} else if ( addr.count == 0xff ) {
 		count = addr.long_count;
 	} else {
-		DBGC2 ( sandev, " <invalid count %#02x>\n", addr.count );
+		DBGC2 ( sandev->drive, " <invalid count %#02x>\n", addr.count );
 		return -INT13_STATUS_INVALID;
 	}
-	DBGC2 ( sandev, " (count %ld)\n", count );
+	DBGC2 ( sandev->drive, " (count %ld)\n", count );
 
 	/* Read from / write to block device */
 	if ( ( rc = sandev_rw ( sandev, lba, count, buffer ) ) != 0 ) {
-		DBGC ( sandev, "INT13 drive %02x extended I/O failed: %s\n",
-		       sandev->drive, strerror ( rc ) );
+		DBGC ( sandev->drive, "INT13 drive %02x extended I/O failed: "
+		       "%s\n", sandev->drive, strerror ( rc ) );
 		/* Record that no blocks were transferred successfully */
 		addr.count = 0;
 		put_real ( addr.count, ix86->segs.ds,
@@ -781,7 +787,7 @@ static int int13_extended_rw ( struct san_device *sandev,
 static int int13_extended_read ( struct san_device *sandev,
 				 struct i386_all_regs *ix86 ) {
 
-	DBGC2 ( sandev, "Extended read: " );
+	DBGC2 ( sandev->drive, "Extended read: " );
 	return int13_extended_rw ( sandev, ix86, sandev_read );
 }
 
@@ -795,7 +801,7 @@ static int int13_extended_read ( struct san_device *sandev,
 static int int13_extended_write ( struct san_device *sandev,
 				  struct i386_all_regs *ix86 ) {
 
-	DBGC2 ( sandev, "Extended write: " );
+	DBGC2 ( sandev->drive, "Extended write: " );
 	return int13_extended_rw ( sandev, ix86, sandev_write );
 }
 
@@ -818,7 +824,7 @@ static int int13_extended_verify ( struct san_device *sandev,
 				 sizeof ( addr ));
 		lba = addr.lba;
 		count = addr.count;
-		DBGC2 ( sandev, "Verify: LBA %08llx (count %ld)\n",
+		DBGC2 ( sandev->drive, "Verify: LBA %08llx (count %ld)\n",
 			( ( unsigned long long ) lba ), count );
 	}
 
@@ -845,7 +851,7 @@ static int int13_extended_seek ( struct san_device *sandev,
 				 sizeof ( addr ));
 		lba = addr.lba;
 		count = addr.count;
-		DBGC2 ( sandev, "Seek: LBA %08llx (count %ld)\n",
+		DBGC2 ( sandev->drive, "Seek: LBA %08llx (count %ld)\n",
 			( ( unsigned long long ) lba ), count );
 	}
 
@@ -879,8 +885,8 @@ static int int13_device_path_info ( struct san_device *sandev,
 	/* Get underlying hardware device */
 	device = identify_device ( &sanpath->block );
 	if ( ! device ) {
-		DBGC ( sandev, "INT13 drive %02x cannot identify hardware "
-		       "device\n", sandev->drive );
+		DBGC ( sandev->drive, "INT13 drive %02x cannot identify "
+		       "hardware device\n", sandev->drive );
 		return -ENODEV;
 	}
 
@@ -895,16 +901,16 @@ static int int13_device_path_info ( struct san_device *sandev,
 		dpi->interface_path.pci.channel = 0xff; /* unused */
 		break;
 	default:
-		DBGC ( sandev, "INT13 drive %02x unrecognised bus type %d\n",
-		       sandev->drive, desc->bus_type );
+		DBGC ( sandev->drive, "INT13 drive %02x unrecognised bus "
+		       "type %d\n", sandev->drive, desc->bus_type );
 		return -ENOTSUP;
 	}
 
 	/* Get EDD block device description */
 	if ( ( rc = edd_describe ( &sanpath->block, &dpi->interface_type,
 				   &dpi->device_path ) ) != 0 ) {
-		DBGC ( sandev, "INT13 drive %02x cannot identify block device: "
-		       "%s\n", sandev->drive, strerror ( rc ) );
+		DBGC ( sandev->drive, "INT13 drive %02x cannot identify "
+		       "block device: %s\n", sandev->drive, strerror ( rc ) );
 		return rc;
 	}
 
@@ -938,8 +944,8 @@ static int int13_get_extended_parameters ( struct san_device *sandev,
 	get_real ( bufsize, ix86->segs.ds,
 		   ( ix86->regs.si + offsetof ( typeof ( params ), bufsize )));
 
-	DBGC2 ( sandev, "Get extended drive parameters to %04x:%04x+%02x\n",
-		ix86->segs.ds, ix86->regs.si, bufsize );
+	DBGC2 ( sandev->drive, "Get extended drive parameters to "
+		"%04x:%04x+%02x\n", ix86->segs.ds, ix86->regs.si, bufsize );
 
 	/* Build drive parameters */
 	memset ( &params, 0, sizeof ( params ) );
@@ -955,8 +961,8 @@ static int int13_get_extended_parameters ( struct san_device *sandev,
 	params.sector_size = sandev_blksize ( sandev );
 	memset ( &params.dpte, 0xff, sizeof ( params.dpte ) );
 	if ( ( rc = int13_device_path_info ( sandev, &params.dpi ) ) != 0 ) {
-		DBGC ( sandev, "INT13 drive %02x could not provide device "
-		       "path information: %s\n",
+		DBGC ( sandev->drive, "INT13 drive %02x could not provide "
+		       "device path information: %s\n",
 		       sandev->drive, strerror ( rc ) );
 		len = offsetof ( typeof ( params ), dpi );
 	}
@@ -973,11 +979,11 @@ static int int13_get_extended_parameters ( struct san_device *sandev,
 		params.bufsize = offsetof ( typeof ( params ), dpi );
 	}
 
-	DBGC ( sandev, "INT 13 drive %02x described using extended "
+	DBGC ( sandev->drive, "INT 13 drive %02x described using extended "
 	       "parameters:\n", sandev->drive );
 	address.segment = ix86->segs.ds;
 	address.offset = ix86->regs.si;
-	DBGC_HDA ( sandev, address, &params, len );
+	DBGC_HDA ( sandev->drive, address, &params, len );
 
 	/* Return drive parameters */
 	if ( len > bufsize )
@@ -998,13 +1004,13 @@ static int int13_cdrom_status_terminate ( struct san_device *sandev,
 					  struct i386_all_regs *ix86 ) {
 	struct int13_cdrom_specification specification;
 
-	DBGC2 ( sandev, "Get CD-ROM emulation status to %04x:%04x%s\n",
+	DBGC2 ( sandev->drive, "Get CD-ROM emulation status to %04x:%04x%s\n",
 		ix86->segs.ds, ix86->regs.si,
 		( ix86->regs.al ? "" : " and terminate" ) );
 
 	/* Fail if we are not a CD-ROM */
 	if ( ! sandev->is_cdrom ) {
-		DBGC ( sandev, "INT13 drive %02x is not a CD-ROM\n",
+		DBGC ( sandev->drive, "INT13 drive %02x is not a CD-ROM\n",
 		       sandev->drive );
 		return -INT13_STATUS_INVALID;
 	}
@@ -1039,11 +1045,12 @@ static int int13_cdrom_read_boot_catalog ( struct san_device *sandev,
 	/* Read parameters from command packet */
 	copy_from_real ( &command, ix86->segs.ds, ix86->regs.si,
 			 sizeof ( command ) );
-	DBGC2 ( sandev, "Read CD-ROM boot catalog to %08x\n", command.buffer );
+	DBGC2 ( sandev->drive, "Read CD-ROM boot catalog to %08x\n",
+		command.buffer );
 
 	/* Fail if we have no boot catalog */
 	if ( ! int13->boot_catalog ) {
-		DBGC ( sandev, "INT13 drive %02x has no boot catalog\n",
+		DBGC ( sandev->drive, "INT13 drive %02x has no boot catalog\n",
 		       sandev->drive );
 		return -INT13_STATUS_INVALID;
 	}
@@ -1052,8 +1059,8 @@ static int int13_cdrom_read_boot_catalog ( struct san_device *sandev,
 	/* Read from boot catalog */
 	if ( ( rc = sandev_read ( sandev, start, command.count,
 				  phys_to_user ( command.buffer ) ) ) != 0 ) {
-		DBGC ( sandev, "INT13 drive %02x could not read boot catalog: "
-		       "%s\n", sandev->drive, strerror ( rc ) );
+		DBGC ( sandev->drive, "INT13 drive %02x could not read boot "
+		       "catalog: %s\n", sandev->drive, strerror ( rc ) );
 		return -INT13_STATUS_READ_ERROR;
 	}
 
@@ -1080,8 +1087,8 @@ static __asmcall __used void int13 ( struct i386_all_regs *ix86 ) {
 		if ( bios_drive != sandev->drive ) {
 			/* Remap any accesses to this drive's natural number */
 			if ( bios_drive == int13->natural_drive ) {
-				DBGC2 ( sandev, "INT13,%02x (%02x) remapped to "
-					"(%02x)\n", ix86->regs.ah,
+				DBGC2 ( sandev->drive, "INT13,%02x (%02x) "
+					"remapped to (%02x)\n", ix86->regs.ah,
 					bios_drive, sandev->drive );
 				ix86->regs.dl = sandev->drive;
 				return;
@@ -1094,7 +1101,7 @@ static __asmcall __used void int13 ( struct i386_all_regs *ix86 ) {
 			}
 		}
 		
-		DBGC2 ( sandev, "INT13,%02x (%02x): ",
+		DBGC2 ( sandev->drive, "INT13,%02x (%02x): ",
 			ix86->regs.ah, bios_drive );
 
 		switch ( command ) {
@@ -1141,7 +1148,7 @@ static __asmcall __used void int13 ( struct i386_all_regs *ix86 ) {
 			status = int13_cdrom_read_boot_catalog ( sandev, ix86 );
 			break;
 		default:
-			DBGC2 ( sandev, "*** Unrecognised INT13 ***\n" );
+			DBGC2 ( sandev->drive, "*** Unrecognised INT13 ***\n" );
 			status = -INT13_STATUS_INVALID;
 			break;
 		}
@@ -1152,8 +1159,9 @@ static __asmcall __used void int13 ( struct i386_all_regs *ix86 ) {
 		/* Negative status indicates an error */
 		if ( status < 0 ) {
 			status = -status;
-			DBGC ( sandev, "INT13,%02x (%02x) failed with status "
-			       "%02x\n", ix86->regs.ah, sandev->drive, status );
+			DBGC ( sandev->drive, "INT13,%02x (%02x) failed with "
+			       "status %02x\n", ix86->regs.ah, sandev->drive,
+			       status );
 		} else {
 			ix86->flags &= ~CF;
 		}
@@ -1269,7 +1277,7 @@ static int int13_hook ( unsigned int drive, struct uri **uris,
 
 	/* Register SAN device */
 	if ( ( rc = register_sandev ( sandev, drive, flags ) ) != 0 ) {
-		DBGC ( sandev, "INT13 drive %02x could not register: %s\n",
+		DBGC ( drive, "INT13 drive %02x could not register: %s\n",
 		       drive, strerror ( rc ) );
 		goto err_register;
 	}
@@ -1289,10 +1297,9 @@ static int int13_hook ( unsigned int drive, struct uri **uris,
 	     ( ( rc = int13_guess_geometry ( sandev, scratch ) ) != 0 ) )
 		goto err_guess_geometry;
 
-	DBGC ( sandev, "INT13 drive %02x (naturally %02x) registered with "
-	       "C/H/S geometry %d/%d/%d\n",
-	       sandev->drive, int13->natural_drive, int13->cylinders,
-	       int13->heads, int13->sectors_per_track );
+	DBGC ( drive, "INT13 drive %02x (naturally %02x) registered with "
+	       "C/H/S geometry %d/%d/%d\n", drive, int13->natural_drive,
+	       int13->cylinders, int13->heads, int13->sectors_per_track );
 
 	/* Hook INT 13 vector if not already hooked */
 	if ( need_hook ) {
@@ -1332,7 +1339,7 @@ static void int13_unhook ( unsigned int drive ) {
 	/* Find drive */
 	sandev = sandev_find ( drive );
 	if ( ! sandev ) {
-		DBG ( "INT13 cannot find drive %02x\n", drive );
+		DBGC ( drive, "INT13 drive %02x is not a SAN drive\n", drive );
 		return;
 	}
 
@@ -1343,7 +1350,7 @@ static void int13_unhook ( unsigned int drive ) {
 	 * to do so reliably.
 	 */
 
-	DBGC ( sandev, "INT13 drive %02x unregistered\n", sandev->drive );
+	DBGC ( drive, "INT13 drive %02x unregistered\n", drive );
 
 	/* Unhook INT 13 vector if no more drives */
 	if ( ! have_sandevs() ) {
@@ -1387,8 +1394,8 @@ static int int13_load_mbr ( unsigned int drive, struct segoff *address ) {
 			       : "a" ( 0x0201 ), "b" ( *address ),
 				 "c" ( 1 ), "d" ( drive ) );
 	if ( status ) {
-		DBG ( "INT13 drive %02x could not read MBR (status %04x)\n",
-		      drive, status );
+		DBGC ( drive, "INT13 drive %02x could not read MBR (status "
+		       "%04x)\n", drive, status );
 		return -EIO;
 	}
 
@@ -1397,8 +1404,8 @@ static int int13_load_mbr ( unsigned int drive, struct segoff *address ) {
 		   ( address->offset +
 		     offsetof ( struct master_boot_record, magic ) ) );
 	if ( magic != INT13_MBR_MAGIC ) {
-		DBG ( "INT13 drive %02x does not contain a valid MBR\n",
-		      drive );
+		DBGC ( drive, "INT13 drive %02x does not contain a valid MBR\n",
+		       drive );
 		return -ENOEXEC;
 	}
 
@@ -1444,8 +1451,8 @@ static int int13_load_eltorito ( unsigned int drive, struct segoff *address ) {
 			       : "a" ( 0x4d00 ), "d" ( drive ),
 				 "S" ( __from_data16 ( &eltorito_cmd ) ) );
 	if ( status ) {
-		DBG ( "INT13 drive %02x could not read El Torito boot catalog "
-		      "(status %04x)\n", drive, status );
+		DBGC ( drive, "INT13 drive %02x could not read El Torito boot "
+		       "catalog (status %04x)\n", drive, status );
 		return -EIO;
 	}
 	copy_from_user ( &catalog, phys_to_user ( eltorito_cmd.buffer ), 0,
@@ -1453,26 +1460,27 @@ static int int13_load_eltorito ( unsigned int drive, struct segoff *address ) {
 
 	/* Sanity checks */
 	if ( catalog.valid.platform_id != ELTORITO_PLATFORM_X86 ) {
-		DBG ( "INT13 drive %02x El Torito specifies unknown platform "
-		      "%02x\n", drive, catalog.valid.platform_id );
+		DBGC ( drive, "INT13 drive %02x El Torito specifies unknown "
+		       "platform %02x\n", drive, catalog.valid.platform_id );
 		return -ENOEXEC;
 	}
 	if ( catalog.boot.indicator != ELTORITO_BOOTABLE ) {
-		DBG ( "INT13 drive %02x El Torito is not bootable\n", drive );
+		DBGC ( drive, "INT13 drive %02x El Torito is not bootable\n",
+		       drive );
 		return -ENOEXEC;
 	}
 	if ( catalog.boot.media_type != ELTORITO_NO_EMULATION ) {
-		DBG ( "INT13 drive %02x El Torito requires emulation "
+		DBGC ( drive, "INT13 drive %02x El Torito requires emulation "
 		       "type %02x\n", drive, catalog.boot.media_type );
 		return -ENOTSUP;
 	}
-	DBG ( "INT13 drive %02x El Torito boot image at LBA %08x (count %d)\n",
-	      drive, catalog.boot.start, catalog.boot.length );
+	DBGC ( drive, "INT13 drive %02x El Torito boot image at LBA %08x "
+	       "(count %d)\n", drive, catalog.boot.start, catalog.boot.length );
 	address->segment = ( catalog.boot.load_segment ?
 			     catalog.boot.load_segment : 0x7c0 );
 	address->offset = 0;
-	DBG ( "INT13 drive %02x El Torito boot image loads at %04x:%04x\n",
-	      drive, address->segment, address->offset );
+	DBGC ( drive, "INT13 drive %02x El Torito boot image loads at "
+	       "%04x:%04x\n", drive, address->segment, address->offset );
 
 	/* Use INT 13, 42 to read the boot image */
 	eltorito_address.bufsize =
@@ -1491,8 +1499,8 @@ static int int13_load_eltorito ( unsigned int drive, struct segoff *address ) {
 			       : "a" ( 0x4200 ), "d" ( drive ),
 				 "S" ( __from_data16 ( &eltorito_address ) ) );
 	if ( status ) {
-		DBG ( "INT13 drive %02x could not read El Torito boot image "
-		      "(status %04x)\n", drive, status );
+		DBGC ( drive, "INT13 drive %02x could not read El Torito boot "
+		       "image (status %04x)\n", drive, status );
 		return -EIO;
 	}
 
@@ -1503,7 +1511,7 @@ static int int13_load_eltorito ( unsigned int drive, struct segoff *address ) {
  * Attempt to boot from an INT 13 drive
  *
  * @v drive		Drive number
- * @v filename		Filename (or NULL to use default)
+ * @v config		Boot configuration parameters
  * @ret rc		Return status code
  *
  * This boots from the specified INT 13 drive by loading the Master
@@ -1513,7 +1521,8 @@ static int int13_load_eltorito ( unsigned int drive, struct segoff *address ) {
  *
  * Note that this function can never return success, by definition.
  */
-static int int13_boot ( unsigned int drive, const char *filename __unused ) {
+static int int13_boot ( unsigned int drive,
+			struct san_boot_config *config __unused ) {
 	struct memory_map memmap;
 	struct segoff address;
 	int rc;
@@ -1533,8 +1542,8 @@ static int int13_boot ( unsigned int drive, const char *filename __unused ) {
 	/* Jump to boot sector */
 	if ( ( rc = call_bootsector ( address.segment, address.offset,
 				      drive ) ) != 0 ) {
-		DBG ( "INT13 drive %02x boot returned: %s\n",
-		      drive, strerror ( rc ) );
+		DBGC ( drive, "INT13 drive %02x boot returned: %s\n",
+		       drive, strerror ( rc ) );
 		return rc;
 	}
 
diff --git a/src/arch/x86/interface/pcbios/pcicloud.c b/src/arch/x86/interface/pcbios/pcicloud.c
index 97d7cac1..98ba38b3 100644
--- a/src/arch/x86/interface/pcbios/pcicloud.c
+++ b/src/arch/x86/interface/pcbios/pcicloud.c
@@ -165,24 +165,27 @@ static void pcicloud_init ( void ) {
 	static struct pci_api *apis[] = {
 		&ecam_api, &pcibios_api, &pcidirect_api
 	};
-	struct pci_range range;
+	struct pci_device pci;
+	uint32_t busdevfn;
 	unsigned int i;
+	int rc;
 
-	/* Select first API that successfully discovers an address range */
+	/* Select first API that successfully discovers a PCI device */
 	for ( i = 0 ; i < ( sizeof ( apis ) / sizeof ( apis[0] ) ) ; i++ ) {
 		pcicloud = apis[i];
-		pcicloud_discover ( 0, &range );
-		if ( range.count != 0 ) {
-			DBGC ( pcicloud, "PCICLOUD selected %s API\n",
-			       pcicloud->name );
-			break;
+		busdevfn = 0;
+		if ( ( rc = pci_find_next ( &pci, &busdevfn ) ) == 0 ) {
+			DBGC ( pcicloud, "PCICLOUD selected %s API (found "
+			       PCI_FMT ")\n", pcicloud->name,
+			       PCI_ARGS ( &pci ) );
+			return;
 		}
 	}
 
-	/* The PCI direct API can never fail discovery since the range
-	 * is hardcoded.
-	 */
-	assert ( range.count != 0 );
+	/* Fall back to using final attempted API if no devices found */
+	pcicloud = apis[ i - 1 ];
+	DBGC ( pcicloud, "PCICLOUD selected %s API (nothing detected)\n",
+	       pcicloud->name );
 }
 
 /** Cloud VM PCI configuration space access initialisation function */
diff --git a/src/arch/x86/interface/pxe/pxe_entry.S b/src/arch/x86/interface/pxe/pxe_entry.S
index 354dd1b3..3899e1bc 100644
--- a/src/arch/x86/interface/pxe/pxe_entry.S
+++ b/src/arch/x86/interface/pxe/pxe_entry.S
@@ -27,6 +27,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #include <librm.h>
 
 	.section ".note.GNU-stack", "", @progbits
+	.code16
 	.arch i386
 
 /****************************************************************************
diff --git a/src/arch/x86/prefix/bootpart.S b/src/arch/x86/prefix/bootpart.S
index 575cb1c0..7b9920fd 100644
--- a/src/arch/x86/prefix/bootpart.S
+++ b/src/arch/x86/prefix/bootpart.S
@@ -6,10 +6,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #define STACK_SIZE	0x2000
 	
 	.section ".note.GNU-stack", "", @progbits
-	.text
+	.code16
 	.arch i386
 	.section ".prefix", "awx", @progbits
-	.code16
 
 /*
  * Find active partition
diff --git a/src/arch/x86/prefix/dskprefix.S b/src/arch/x86/prefix/dskprefix.S
index bc194887..e8e55ef1 100644
--- a/src/arch/x86/prefix/dskprefix.S
+++ b/src/arch/x86/prefix/dskprefix.S
@@ -26,10 +26,9 @@ FILE_LICENCE ( GPL2_ONLY )
 
 	.section ".note.GNU-stack", "", @progbits
 	.org	0
+	.code16
 	.arch i386
-	.text
 	.section ".prefix", "ax", @progbits
-	.code16
 	.globl	_dsk_start
 _dsk_start:
 
diff --git a/src/arch/x86/prefix/exeprefix.S b/src/arch/x86/prefix/exeprefix.S
index 5b2605e8..98ed6c5f 100644
--- a/src/arch/x86/prefix/exeprefix.S
+++ b/src/arch/x86/prefix/exeprefix.S
@@ -37,10 +37,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #define PSP_CMDLINE_START 0x81
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
+	.code16
 	.arch i386
 	.org 0
-	.code16
 	.section ".prefix", "awx", @progbits
 
 signature:
diff --git a/src/arch/x86/prefix/hdprefix.S b/src/arch/x86/prefix/hdprefix.S
index fbf8d2e4..3133dec6 100644
--- a/src/arch/x86/prefix/hdprefix.S
+++ b/src/arch/x86/prefix/hdprefix.S
@@ -3,10 +3,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #include <librm.h>
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
+	.code16
 	.arch i386
 	.section ".prefix", "awx", @progbits
-	.code16
 	.org 0
 	.globl	_hd_start
 _hd_start:
diff --git a/src/arch/x86/prefix/libprefix.S b/src/arch/x86/prefix/libprefix.S
index 380e471d..b08a5782 100644
--- a/src/arch/x86/prefix/libprefix.S
+++ b/src/arch/x86/prefix/libprefix.S
@@ -27,6 +27,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #include <librm.h>
 
 	.section ".note.GNU-stack", "", @progbits
+	.code16
 	.arch i386
 
 /* Image compression enabled */
diff --git a/src/arch/x86/prefix/lkrnprefix.S b/src/arch/x86/prefix/lkrnprefix.S
index 2c17f79d..c8a04c9d 100644
--- a/src/arch/x86/prefix/lkrnprefix.S
+++ b/src/arch/x86/prefix/lkrnprefix.S
@@ -5,9 +5,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #define BZI_LOAD_HIGH_ADDR 0x100000
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
-	.arch i386
 	.code16
+	.arch i386
 	.section ".prefix", "ax", @progbits
 	.globl	_lkrn_start
 _lkrn_start:
diff --git a/src/arch/x86/prefix/mbr.S b/src/arch/x86/prefix/mbr.S
index 928bb338..5e0ed5dd 100644
--- a/src/arch/x86/prefix/mbr.S
+++ b/src/arch/x86/prefix/mbr.S
@@ -1,10 +1,9 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
+	.code16
 	.arch i386
 	.section ".prefix", "awx", @progbits
-	.code16
 	.org 0
 
 	.globl mbr
diff --git a/src/arch/x86/prefix/mromprefix.S b/src/arch/x86/prefix/mromprefix.S
index 5f3496b2..d05278e6 100644
--- a/src/arch/x86/prefix/mromprefix.S
+++ b/src/arch/x86/prefix/mromprefix.S
@@ -42,9 +42,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #include "pciromprefix.S"
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
-	.arch i386
 	.code16
+	.arch i386
 
 /* Obtain access to payload by exposing the expansion ROM BAR at the
  * address currently used by a suitably large memory BAR on the same
diff --git a/src/arch/x86/prefix/nbiprefix.S b/src/arch/x86/prefix/nbiprefix.S
index cae1009b..bbacd4b7 100644
--- a/src/arch/x86/prefix/nbiprefix.S
+++ b/src/arch/x86/prefix/nbiprefix.S
@@ -3,9 +3,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #include <librm.h>
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
-	.arch i386
 	.code16
+	.arch i386
 	.section ".prefix", "ax", @progbits
 	.org 0
 
diff --git a/src/arch/x86/prefix/nullprefix.S b/src/arch/x86/prefix/nullprefix.S
index 1568188d..426f1f2c 100644
--- a/src/arch/x86/prefix/nullprefix.S
+++ b/src/arch/x86/prefix/nullprefix.S
@@ -2,11 +2,10 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
 	.org	0
-	.text
+	.code16
 	.arch i386
 
 	.section ".prefix", "ax", @progbits
-	.code16
 _prefix:
 
 	.section ".text16", "ax", @progbits
diff --git a/src/arch/x86/prefix/pxeprefix.S b/src/arch/x86/prefix/pxeprefix.S
index 494fbc13..5181ef61 100644
--- a/src/arch/x86/prefix/pxeprefix.S
+++ b/src/arch/x86/prefix/pxeprefix.S
@@ -12,10 +12,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #define PXE_HACK_EB54			0x0001
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
+	.code16
 	.arch i386
 	.org 0
-	.code16
 
 #include <librm.h>
 #include <undi.h>
diff --git a/src/arch/x86/prefix/rawprefix.S b/src/arch/x86/prefix/rawprefix.S
index 4a3d3504..962c9718 100644
--- a/src/arch/x86/prefix/rawprefix.S
+++ b/src/arch/x86/prefix/rawprefix.S
@@ -9,10 +9,9 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
+	.code16
 	.arch i386
 	.org 0
-	.code16
 
 #include <librm.h>
 
diff --git a/src/arch/x86/prefix/romprefix.S b/src/arch/x86/prefix/romprefix.S
index 79fed2a3..09837cee 100644
--- a/src/arch/x86/prefix/romprefix.S
+++ b/src/arch/x86/prefix/romprefix.S
@@ -55,7 +55,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #endif
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
 	.code16
 	.arch i386
 	.section ".prefix", "ax", @progbits
diff --git a/src/arch/x86/prefix/undiloader.S b/src/arch/x86/prefix/undiloader.S
index e544d504..33573230 100644
--- a/src/arch/x86/prefix/undiloader.S
+++ b/src/arch/x86/prefix/undiloader.S
@@ -3,7 +3,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #include <librm.h>
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
 	.code16
 	.arch i386
 	.section ".prefix", "ax", @progbits
diff --git a/src/arch/x86/prefix/unlzma.S b/src/arch/x86/prefix/unlzma.S
index f4bd81bd..e4d1e190 100644
--- a/src/arch/x86/prefix/unlzma.S
+++ b/src/arch/x86/prefix/unlzma.S
@@ -44,7 +44,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  */
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
+	.code32
 	.arch i486
 	.section ".prefix.lib", "ax", @progbits
 
diff --git a/src/arch/x86/prefix/usbdisk.S b/src/arch/x86/prefix/usbdisk.S
index 461a0837..11ab6a46 100644
--- a/src/arch/x86/prefix/usbdisk.S
+++ b/src/arch/x86/prefix/usbdisk.S
@@ -3,10 +3,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #include <config/console.h>
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
+	.code16
 	.arch i386
 	.section ".prefix", "awx", @progbits
-	.code16
 	.org 0
 
 #include "mbr.S"
diff --git a/src/arch/x86/transitions/liba20.S b/src/arch/x86/transitions/liba20.S
index 6c1bac67..971cff22 100644
--- a/src/arch/x86/transitions/liba20.S
+++ b/src/arch/x86/transitions/liba20.S
@@ -25,6 +25,7 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.section ".note.GNU-stack", "", @progbits
+	.code16
 	.arch i386
 
 /****************************************************************************
diff --git a/src/arch/x86/transitions/libkir.S b/src/arch/x86/transitions/libkir.S
index af090b26..2c4dc948 100644
--- a/src/arch/x86/transitions/libkir.S
+++ b/src/arch/x86/transitions/libkir.S
@@ -32,10 +32,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 #define BOCHSBP xchgw %bx, %bx
 
 	.section ".note.GNU-stack", "", @progbits
-	.text
+	.code16
 	.arch i386
 	.section ".text16", "awx", @progbits
-	.code16
 	
 /****************************************************************************
  * init_libkir (real-mode or 16:xx protected-mode far call)
diff --git a/src/bin/.gitignore b/src/bin/.gitignore
index 72e8ffc0..d6b7ef32 100644
--- a/src/bin/.gitignore
+++ b/src/bin/.gitignore
@@ -1 +1,2 @@
 *
+!.gitignore
diff --git a/src/config/config_crypto.c b/src/config/config_crypto.c
index fa1996a5..5211224a 100644
--- a/src/config/config_crypto.c
+++ b/src/config/config_crypto.c
@@ -83,6 +83,11 @@ REQUIRE_OBJECT ( oid_sha512_224 );
 REQUIRE_OBJECT ( oid_sha512_256 );
 #endif
 
+/* X25519 */
+#if defined ( CRYPTO_CURVE_X25519 )
+REQUIRE_OBJECT ( oid_x25519 );
+#endif
+
 /* RSA and MD5 */
 #if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_DIGEST_MD5 )
 REQUIRE_OBJECT ( rsa_md5 );
@@ -114,25 +119,79 @@ REQUIRE_OBJECT ( rsa_sha512 );
 #endif
 
 /* RSA, AES-CBC, and SHA-1 */
-#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_CIPHER_AES_CBC ) && \
-    defined ( CRYPTO_DIGEST_SHA1 )
+#if defined ( CRYPTO_EXCHANGE_PUBKEY ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_CBC ) && defined ( CRYPTO_DIGEST_SHA1 )
 REQUIRE_OBJECT ( rsa_aes_cbc_sha1 );
 #endif
 
 /* RSA, AES-CBC, and SHA-256 */
-#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_CIPHER_AES_CBC ) && \
-    defined ( CRYPTO_DIGEST_SHA256 )
+#if defined ( CRYPTO_EXCHANGE_PUBKEY ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_CBC ) && defined ( CRYPTO_DIGEST_SHA256 )
 REQUIRE_OBJECT ( rsa_aes_cbc_sha256 );
 #endif
 
 /* RSA, AES-GCM, and SHA-256 */
-#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_CIPHER_AES_GCM ) && \
-    defined ( CRYPTO_DIGEST_SHA256 )
+#if defined ( CRYPTO_EXCHANGE_PUBKEY ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_GCM ) && defined ( CRYPTO_DIGEST_SHA256 )
 REQUIRE_OBJECT ( rsa_aes_gcm_sha256 );
 #endif
 
 /* RSA, AES-GCM, and SHA-384 */
-#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_CIPHER_AES_GCM ) && \
-    defined ( CRYPTO_DIGEST_SHA384 )
+#if defined ( CRYPTO_EXCHANGE_PUBKEY ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_GCM ) && defined ( CRYPTO_DIGEST_SHA384 )
 REQUIRE_OBJECT ( rsa_aes_gcm_sha384 );
 #endif
+
+/* DHE, RSA, AES-CBC, and SHA-1 */
+#if defined ( CRYPTO_EXCHANGE_DHE ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_CBC ) && defined ( CRYPTO_DIGEST_SHA1 )
+REQUIRE_OBJECT ( dhe_rsa_aes_cbc_sha1 );
+#endif
+
+/* DHE, RSA, AES-CBC, and SHA-256 */
+#if defined ( CRYPTO_EXCHANGE_DHE ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_CBC ) && defined ( CRYPTO_DIGEST_SHA256 )
+REQUIRE_OBJECT ( dhe_rsa_aes_cbc_sha256 );
+#endif
+
+/* DHE, RSA, AES-GCM, and SHA-256 */
+#if defined ( CRYPTO_EXCHANGE_DHE ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_GCM ) && defined ( CRYPTO_DIGEST_SHA256 )
+REQUIRE_OBJECT ( dhe_rsa_aes_gcm_sha256 );
+#endif
+
+/* DHE, RSA, AES-GCM, and SHA-384 */
+#if defined ( CRYPTO_EXCHANGE_DHE ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_GCM ) && defined ( CRYPTO_DIGEST_SHA384 )
+REQUIRE_OBJECT ( dhe_rsa_aes_gcm_sha384 );
+#endif
+
+/* ECDHE, RSA, AES-CBC, and SHA-1 */
+#if defined ( CRYPTO_EXCHANGE_ECDHE ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_CBC ) && defined ( CRYPTO_DIGEST_SHA1 )
+REQUIRE_OBJECT ( ecdhe_rsa_aes_cbc_sha1 );
+#endif
+
+/* ECDHE, RSA, AES-CBC, and SHA-256 */
+#if defined ( CRYPTO_EXCHANGE_ECDHE ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_CBC ) && defined ( CRYPTO_DIGEST_SHA256 )
+REQUIRE_OBJECT ( ecdhe_rsa_aes_cbc_sha256 );
+#endif
+
+/* ECDHE, RSA, AES-CBC, and SHA-384 */
+#if defined ( CRYPTO_EXCHANGE_ECDHE ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_CBC ) && defined ( CRYPTO_DIGEST_SHA384 )
+REQUIRE_OBJECT ( ecdhe_rsa_aes_cbc_sha384 );
+#endif
+
+/* ECDHE, RSA, AES-GCM, and SHA-256 */
+#if defined ( CRYPTO_EXCHANGE_ECDHE ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_GCM ) && defined ( CRYPTO_DIGEST_SHA256 )
+REQUIRE_OBJECT ( ecdhe_rsa_aes_gcm_sha256 );
+#endif
+
+/* ECDHE, RSA, AES-GCM, and SHA-384 */
+#if defined ( CRYPTO_EXCHANGE_ECDHE ) && defined ( CRYPTO_PUBKEY_RSA ) && \
+    defined ( CRYPTO_CIPHER_AES_GCM ) && defined ( CRYPTO_DIGEST_SHA384 )
+REQUIRE_OBJECT ( ecdhe_rsa_aes_gcm_sha384 );
+#endif
diff --git a/src/config/config_eap.c b/src/config/config_eap.c
new file mode 100644
index 00000000..e18c48ca
--- /dev/null
+++ b/src/config/config_eap.c
@@ -0,0 +1,42 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <config/general.h>
+
+/** @file
+ *
+ * EAP configuration options
+ *
+ */
+
+PROVIDE_REQUIRING_SYMBOL();
+
+/*
+ * Drag in EAP authentication methods
+ */
+#ifdef EAP_METHOD_MD5
+REQUIRE_OBJECT ( eap_md5 );
+#endif
+#ifdef EAP_METHOD_MSCHAPV2
+REQUIRE_OBJECT ( eap_mschapv2 );
+#endif
diff --git a/src/config/crypto.h b/src/config/crypto.h
index 76bf14d4..589c4f0d 100644
--- a/src/config/crypto.h
+++ b/src/config/crypto.h
@@ -12,6 +12,15 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 /** Minimum TLS version */
 #define TLS_VERSION_MIN TLS_VERSION_TLS_1_1
 
+/** Public-key exchange algorithm */
+#define CRYPTO_EXCHANGE_PUBKEY
+
+/** DHE key exchange algorithm */
+#define CRYPTO_EXCHANGE_DHE
+
+/** ECDHE key exchange algorithm */
+#define CRYPTO_EXCHANGE_ECDHE
+
 /** RSA public-key algorithm */
 #define CRYPTO_PUBKEY_RSA
 
@@ -48,6 +57,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 /** SHA-512/256 digest algorithm */
 //#define CRYPTO_DIGEST_SHA512_256
 
+/** X25519 elliptic curve */
+#define CRYPTO_CURVE_X25519
+
 /** Margin of error (in seconds) allowed in signed timestamps
  *
  * We default to allowing a reasonable margin of error: 12 hours to
diff --git a/src/config/general.h b/src/config/general.h
index fff4b392..bcf7e69c 100644
--- a/src/config/general.h
+++ b/src/config/general.h
@@ -92,6 +92,13 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define	CRYPTO_80211_WPA2	/* Add support for stronger WPA cryptography */
 
 /*
+ * 802.1x EAP authentication methods
+ *
+ */
+#define EAP_METHOD_MD5		/* MD5-Challenge port authentication */
+//#define EAP_METHOD_MSCHAPV2	/* MS-CHAPv2 port authentication */
+
+/*
  * Name resolution modules
  *
  */
diff --git a/src/config/local/.gitignore b/src/config/local/.gitignore
index 72e8ffc0..d6b7ef32 100644
--- a/src/config/local/.gitignore
+++ b/src/config/local/.gitignore
@@ -1 +1,2 @@
 *
+!.gitignore
diff --git a/src/core/base16.c b/src/core/base16.c
index f9e0f336..47e35f41 100644
--- a/src/core/base16.c
+++ b/src/core/base16.c
@@ -78,12 +78,23 @@ int hex_decode ( char separator, const char *encoded, void *data, size_t len ) {
 	unsigned int count = 0;
 	unsigned int sixteens;
 	unsigned int units;
+	int optional;
 
+	/* Strip out optionality flag from separator character */
+	optional = ( separator & HEX_DECODE_OPTIONAL );
+	separator &= ~HEX_DECODE_OPTIONAL;
+
+	/* Decode string */
 	while ( *encoded ) {
 
 		/* Check separator, if applicable */
-		if ( count && separator && ( ( *(encoded++) != separator ) ) )
-			return -EINVAL;
+		if ( count && separator ) {
+			if ( *encoded == separator ) {
+				encoded++;
+			} else if ( ! optional ) {
+				return -EINVAL;
+			}
+		}
 
 		/* Extract digits.  Note that either digit may be NUL,
 		 * which would be interpreted as an invalid value by
diff --git a/src/core/dummy_sanboot.c b/src/core/dummy_sanboot.c
index e6293099..e22998da 100644
--- a/src/core/dummy_sanboot.c
+++ b/src/core/dummy_sanboot.c
@@ -55,7 +55,7 @@ static int dummy_san_hook ( unsigned int drive, struct uri **uris,
 
 	/* Register SAN device */
 	if ( ( rc = register_sandev ( sandev, drive, flags ) ) != 0 ) {
-		DBGC ( sandev, "SAN %#02x could not register: %s\n",
+		DBGC ( sandev->drive, "SAN %#02x could not register: %s\n",
 		       sandev->drive, strerror ( rc ) );
 		goto err_register;
 	}
@@ -80,7 +80,7 @@ static void dummy_san_unhook ( unsigned int drive ) {
 	/* Find drive */
 	sandev = sandev_find ( drive );
 	if ( ! sandev ) {
-		DBG ( "SAN %#02x does not exist\n", drive );
+		DBGC ( drive, "SAN %#02x does not exist\n", drive );
 		return;
 	}
 
@@ -95,11 +95,11 @@ static void dummy_san_unhook ( unsigned int drive ) {
  * Boot from dummy SAN device
  *
  * @v drive		Drive number
- * @v filename		Filename (or NULL to use default)
+ * @v config		Boot configuration parameters
  * @ret rc		Return status code
  */
 static int dummy_san_boot ( unsigned int drive __unused,
-			    const char *filename __unused ) {
+			    struct san_boot_config *config __unused ) {
 
 	return -EOPNOTSUPP;
 }
diff --git a/src/core/null_sanboot.c b/src/core/null_sanboot.c
index 7c0680f5..2340cd2a 100644
--- a/src/core/null_sanboot.c
+++ b/src/core/null_sanboot.c
@@ -38,7 +38,7 @@ static void null_san_unhook ( unsigned int drive __unused ) {
 }
 
 static int null_san_boot ( unsigned int drive __unused,
-			   const char *filename __unused ) {
+			   struct san_boot_config *config __unused ) {
 	return -EOPNOTSUPP;
 }
 
diff --git a/src/core/parseopt.c b/src/core/parseopt.c
index cd3b3101..8410e6e9 100644
--- a/src/core/parseopt.c
+++ b/src/core/parseopt.c
@@ -31,6 +31,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ctype.h>
 #include <errno.h>
 #include <getopt.h>
+#include <ipxe/uuid.h>
 #include <ipxe/netdevice.h>
 #include <ipxe/menu.h>
 #include <ipxe/settings.h>
@@ -125,6 +126,29 @@ int parse_timeout ( char *text, unsigned long *value ) {
 }
 
 /**
+ * Parse UUID
+ *
+ * @v text		Text
+ * @ret uuid		UUID value
+ * @ret rc		Return status code
+ */
+int parse_uuid ( char *text, struct uuid_option *uuid ) {
+	int rc;
+
+	/* Sanity check */
+	assert ( text != NULL );
+
+	/* Parse UUID */
+	if ( ( rc = uuid_aton ( text, &uuid->buf ) ) != 0 ) {
+		printf ( "\"%s\": invalid UUID\n", text );
+		return rc;
+	}
+	uuid->value = &uuid->buf;
+
+	return 0;
+}
+
+/**
  * Parse network device name
  *
  * @v text		Text
diff --git a/src/core/sanboot.c b/src/core/sanboot.c
index cabc4843..e49a3f92 100644
--- a/src/core/sanboot.c
+++ b/src/core/sanboot.c
@@ -45,16 +45,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/sanboot.h>
 
 /**
- * Default SAN drive number
- *
- * The drive number is a meaningful concept only in a BIOS
- * environment, where it represents the INT13 drive number (0x80 for
- * the first hard disk).  We retain it in other environments to allow
- * for a simple way for iPXE commands to refer to SAN drives.
- */
-#define SAN_DEFAULT_DRIVE 0x80
-
-/**
  * Timeout for block device commands (in ticks)
  *
  * Underlying devices should ideally never become totally stuck.
@@ -108,6 +98,22 @@ struct san_device * sandev_find ( unsigned int drive ) {
 }
 
 /**
+ * Find next SAN device by drive number
+ *
+ * @v drive		Minimum drive number
+ * @ret sandev		SAN device, or NULL
+ */
+struct san_device * sandev_next ( unsigned int drive ) {
+	struct san_device *sandev;
+
+	list_for_each_entry ( sandev, &san_devices, list ) {
+		if ( sandev->drive >= drive )
+			return sandev;
+	}
+	return NULL;
+}
+
+/**
  * Free SAN device
  *
  * @v refcnt		Reference count
@@ -197,7 +203,7 @@ static int sanpath_open ( struct san_path *sanpath ) {
 
 	/* Open interface */
 	if ( ( rc = xfer_open_uri ( &sanpath->block, sanpath->uri ) ) != 0 ) {
-		DBGC ( sandev, "SAN %#02x.%d could not (re)open URI: "
+		DBGC ( sandev->drive, "SAN %#02x.%d could not (re)open URI: "
 		       "%s\n", sandev->drive, sanpath->index, strerror ( rc ) );
 		return rc;
 	}
@@ -265,7 +271,7 @@ static void sanpath_block_close ( struct san_path *sanpath, int rc ) {
 	/* Any closure is an error from our point of view */
 	if ( rc == 0 )
 		rc = -ENOTCONN;
-	DBGC ( sandev, "SAN %#02x.%d closed: %s\n",
+	DBGC ( sandev->drive, "SAN %#02x.%d closed: %s\n",
 	       sandev->drive, sanpath->index, strerror ( rc ) );
 
 	/* Close path */
@@ -307,11 +313,11 @@ static void sanpath_step ( struct san_path *sanpath ) {
 
 	/* Mark as active path or close as applicable */
 	if ( ! sandev->active ) {
-		DBGC ( sandev, "SAN %#02x.%d is active\n",
+		DBGC ( sandev->drive, "SAN %#02x.%d is active\n",
 		       sandev->drive, sanpath->index );
 		sandev->active = sanpath;
 	} else {
-		DBGC ( sandev, "SAN %#02x.%d is available\n",
+		DBGC ( sandev->drive, "SAN %#02x.%d is available\n",
 		       sandev->drive, sanpath->index );
 		sanpath_close ( sanpath, 0 );
 	}
@@ -398,8 +404,9 @@ int sandev_reopen ( struct san_device *sandev ) {
 				rc = sanpath->path_rc;
 				break;
 			}
-			DBGC ( sandev, "SAN %#02x never became available: %s\n",
-			       sandev->drive, strerror ( rc ) );
+			DBGC ( sandev->drive, "SAN %#02x never became "
+			       "available: %s\n", sandev->drive,
+			       strerror ( rc ) );
 			goto err_none;
 		}
 	}
@@ -453,8 +460,9 @@ static int sandev_command_rw ( struct san_device *sandev,
 	if ( ( rc = params->rw.block_rw ( &sanpath->block, &sandev->command,
 					  params->rw.lba, params->rw.count,
 					  params->rw.buffer, len ) ) != 0 ) {
-		DBGC ( sandev, "SAN %#02x.%d could not initiate read/write: "
-		       "%s\n", sandev->drive, sanpath->index, strerror ( rc ) );
+		DBGC ( sandev->drive, "SAN %#02x.%d could not initiate "
+		       "read/write: %s\n", sandev->drive, sanpath->index,
+		       strerror ( rc ) );
 		return rc;
 	}
 
@@ -480,8 +488,9 @@ sandev_command_read_capacity ( struct san_device *sandev,
 	/* Initiate read capacity command */
 	if ( ( rc = block_read_capacity ( &sanpath->block,
 					  &sandev->command ) ) != 0 ) {
-		DBGC ( sandev, "SAN %#02x.%d could not initiate read capacity: "
-		       "%s\n", sandev->drive, sanpath->index, strerror ( rc ) );
+		DBGC ( sandev->drive, "SAN %#02x.%d could not initiate read "
+		       "capacity: %s\n", sandev->drive, sanpath->index,
+		       strerror ( rc ) );
 		return rc;
 	}
 
@@ -565,7 +574,7 @@ sandev_command ( struct san_device *sandev,
 int sandev_reset ( struct san_device *sandev ) {
 	int rc;
 
-	DBGC ( sandev, "SAN %#02x reset\n", sandev->drive );
+	DBGC ( sandev->drive, "SAN %#02x reset\n", sandev->drive );
 
 	/* Close and reopen underlying block device */
 	if ( ( rc = sandev_reopen ( sandev ) ) != 0 )
@@ -698,8 +707,8 @@ static int sandev_describe ( struct san_device *sandev ) {
 			if ( ! desc )
 				continue;
 			if ( ( rc = desc->model->complete ( desc ) ) != 0 ) {
-				DBGC ( sandev, "SAN %#02x.%d could not be "
-				       "described: %s\n", sandev->drive,
+				DBGC ( sandev->drive, "SAN %#02x.%d could not "
+				       "be described: %s\n", sandev->drive,
 				       sanpath->index, strerror ( rc ) );
 				return rc;
 			}
@@ -792,8 +801,8 @@ static int sandev_parse_iso9660 ( struct san_device *sandev ) {
 	/* Read primary volume descriptor */
 	if ( ( rc = sandev_read ( sandev, lba, count,
 				  virt_to_user ( scratch ) ) ) != 0 ) {
-		DBGC ( sandev, "SAN %#02x could not read ISO9660 primary"
-		       "volume descriptor: %s\n",
+		DBGC ( sandev->drive, "SAN %#02x could not read ISO9660 "
+		       "primary volume descriptor: %s\n",
 		       sandev->drive, strerror ( rc ) );
 		goto err_rw;
 	}
@@ -801,8 +810,8 @@ static int sandev_parse_iso9660 ( struct san_device *sandev ) {
 	/* Configure as CD-ROM if applicable */
 	if ( memcmp ( &scratch->primary.fixed, &primary_check,
 		      sizeof ( primary_check ) ) == 0 ) {
-		DBGC ( sandev, "SAN %#02x contains an ISO9660 filesystem; "
-		       "treating as CD-ROM\n", sandev->drive );
+		DBGC ( sandev->drive, "SAN %#02x contains an ISO9660 "
+		       "filesystem; treating as CD-ROM\n", sandev->drive );
 		sandev->blksize_shift = blksize_shift;
 		sandev->is_cdrom = 1;
 	}
@@ -867,11 +876,12 @@ struct san_device * alloc_sandev ( struct uri **uris, unsigned int count,
  */
 int register_sandev ( struct san_device *sandev, unsigned int drive,
 		      unsigned int flags ) {
+	struct san_device *before;
 	int rc;
 
 	/* Check that drive number is not in use */
 	if ( sandev_find ( drive ) != NULL ) {
-		DBGC ( sandev, "SAN %#02x is already in use\n", drive );
+		DBGC ( sandev->drive, "SAN %#02x is already in use\n", drive );
 		rc = -EADDRINUSE;
 		goto err_in_use;
 	}
@@ -900,9 +910,13 @@ int register_sandev ( struct san_device *sandev, unsigned int drive,
 	if ( ( rc = sandev_parse_iso9660 ( sandev ) ) != 0 )
 		goto err_iso9660;
 
-	/* Add to list of SAN devices */
-	list_add_tail ( &sandev->list, &san_devices );
-	DBGC ( sandev, "SAN %#02x registered\n", sandev->drive );
+	/* Add to list of SAN devices, in drive order */
+	for_each_sandev ( before ) {
+		if ( before->drive > sandev->drive )
+			break;
+	}
+	list_add_tail ( &sandev->list, &before->list );
+	DBGC ( sandev->drive, "SAN %#02x registered\n", sandev->drive );
 
 	return 0;
 
@@ -936,7 +950,7 @@ void unregister_sandev ( struct san_device *sandev ) {
 	/* Remove ACPI descriptors */
 	sandev_undescribe ( sandev );
 
-	DBGC ( sandev, "SAN %#02x unregistered\n", sandev->drive );
+	DBGC ( sandev->drive, "SAN %#02x unregistered\n", sandev->drive );
 }
 
 /** The "san-drive" setting */
diff --git a/src/core/settings.c b/src/core/settings.c
index 1037b06a..bece1afe 100644
--- a/src/core/settings.c
+++ b/src/core/settings.c
@@ -2242,6 +2242,37 @@ const struct setting_type setting_type_base64 __setting_type = {
 };
 
 /**
+ * Parse UUID/GUID setting value
+ *
+ * @v type		Setting type
+ * @v value		Formatted setting value
+ * @v buf		Buffer to contain raw value
+ * @v len		Length of buffer
+ * @v size		Integer size, in bytes
+ * @ret len		Length of raw value, or negative error
+ */
+static int parse_uuid_setting ( const struct setting_type *type,
+				const char *value, void *buf, size_t len ) {
+	union uuid uuid;
+	int rc;
+
+	/* Parse UUID */
+	if ( ( rc = uuid_aton ( value, &uuid ) ) != 0 )
+		return rc;
+
+	/* Mangle GUID byte ordering */
+	if ( type == &setting_type_guid )
+		uuid_mangle ( &uuid );
+
+	/* Copy value */
+	if ( len > sizeof ( uuid ) )
+		len = sizeof ( uuid );
+	memcpy ( buf, uuid.raw, len );
+
+	return ( sizeof ( uuid ) );
+}
+
+/**
  * Format UUID/GUID setting value
  *
  * @v type		Setting type
@@ -2274,12 +2305,14 @@ static int format_uuid_setting ( const struct setting_type *type,
 /** UUID setting type */
 const struct setting_type setting_type_uuid __setting_type = {
 	.name = "uuid",
+	.parse = parse_uuid_setting,
 	.format = format_uuid_setting,
 };
 
 /** GUID setting type */
 const struct setting_type setting_type_guid __setting_type = {
 	.name = "guid",
+	.parse = parse_uuid_setting,
 	.format = format_uuid_setting,
 };
 
diff --git a/src/core/uuid.c b/src/core/uuid.c
index c43d4216..b6600af7 100644
--- a/src/core/uuid.c
+++ b/src/core/uuid.c
@@ -25,7 +25,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdio.h>
+#include <errno.h>
 #include <byteswap.h>
+#include <ipxe/base16.h>
 #include <ipxe/uuid.h>
 
 /** @file
@@ -53,3 +55,29 @@ const char * uuid_ntoa ( const union uuid *uuid ) {
 		  uuid->canonical.e[4], uuid->canonical.e[5] );
 	return buf;
 }
+
+/**
+ * Parse UUID
+ *
+ * @v string		UUID string
+ * @v uuid		UUID to fill in
+ * @ret rc		Return status code
+ */
+int uuid_aton ( const char *string, union uuid *uuid ) {
+	int len;
+	int rc;
+
+	/* Decode as hex string with optional '-' separator */
+	len = hex_decode ( ( '-' | HEX_DECODE_OPTIONAL ), string, uuid->raw,
+			   sizeof ( *uuid ) );
+	if ( len < 0 ) {
+		rc = len;
+		return rc;
+	}
+
+	/* Check length */
+	if ( len != sizeof ( *uuid ) )
+		return -EINVAL;
+
+	return 0;
+}
diff --git a/src/crypto/bigint.c b/src/crypto/bigint.c
index ac9670ef..656f979e 100644
--- a/src/crypto/bigint.c
+++ b/src/crypto/bigint.c
@@ -51,6 +51,31 @@ static struct profiler bigint_mod_multiply_subtract_profiler __profiler =
 	{ .name = "bigint_mod_multiply.subtract" };
 
 /**
+ * Conditionally swap big integers (in constant time)
+ *
+ * @v first0		Element 0 of big integer to be conditionally swapped
+ * @v second0		Element 0 of big integer to be conditionally swapped
+ * @v size		Number of elements in big integers
+ * @v swap		Swap first and second big integers
+ */
+void bigint_swap_raw ( bigint_element_t *first0, bigint_element_t *second0,
+		       unsigned int size, int swap ) {
+	bigint_element_t mask;
+	bigint_element_t xor;
+	unsigned int i;
+
+	/* Construct mask */
+	mask = ( ( bigint_element_t ) ( ! swap ) - 1 );
+
+	/* Conditionally swap elements */
+	for ( i = 0 ; i < size ; i++ ) {
+		xor = ( mask & ( first0[i] ^ second0[i] ) );
+		first0[i] ^= xor;
+		second0[i] ^= xor;
+	}
+}
+
+/**
  * Perform modular multiplication of big integers
  *
  * @v multiplicand0	Element 0 of big integer to be multiplied
diff --git a/src/crypto/des.c b/src/crypto/des.c
new file mode 100644
index 00000000..6918bec3
--- /dev/null
+++ b/src/crypto/des.c
@@ -0,0 +1,695 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * DES algorithm
+ *
+ * DES was not designed to be implemented in software, and therefore
+ * contains a large number of bit permutation operations that are
+ * essentially free in hardware (requiring only wires, no gates) but
+ * expensive in software.
+ *
+ * Since DES is no longer used as a practical block cipher for large
+ * volumes of data, we optimise for code size, and do not attempt to
+ * obtain fast throughput.
+ *
+ * The algorithm is specified in NIST SP 800-67, downloadable from
+ * https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-67r2.pdf
+ */
+
+#include <stdint.h>
+#include <string.h>
+#include <errno.h>
+#include <byteswap.h>
+#include <ipxe/rotate.h>
+#include <ipxe/crypto.h>
+#include <ipxe/ecb.h>
+#include <ipxe/cbc.h>
+#include <ipxe/init.h>
+#include <ipxe/des.h>
+
+/**
+ * DES shift schedule
+ *
+ * The DES shift schedule (ordered from round 16 down to round 1) is
+ * {1,2,2,2,2,2,2,1,2,2,2,2,2,2,1,1}.  In binary, this may be
+ * represented as {1,10,10,10,10,10,10,1,10,10,10,10,10,10,1,1} and
+ * concatenated (without padding) to produce a single binary integer
+ * 1101010101010110101010101011 (equal to 0x0d556aab in hexadecimal).
+ *
+ * This integer may then be consumed LSB-first, where a 1 bit
+ * indicates a shift and the generation of a round key, and a 0 bit
+ * indicates a shift without the generation of a round key.
+ */
+#define DES_SCHEDULE 0x0d556aab
+
+/**
+ * Define an element pair in a DES S-box
+ *
+ * @v x			Upper element of element pair
+ * @v y			Lower element of element pair
+ *
+ * DES S-box elements are 4-bit values.  We encode two values per
+ * byte, ordering the elements so that the six-bit input value may be
+ * used directly as a lookup index.
+ *
+ * Specifically, if the input value is {r1,c3,c2,c1,c0,r0}, where
+ * {r1,r0} is the table row index and {c3,c2,c1,c0} is the table
+ * column index (as used in the DES specification), then:
+ *
+ *   - {r1,c3,c2,c1,c0} is the byte index into the table
+ *
+ *   - (4*r0) is the required bit shift to extract the 4-bit value
+ */
+#define SBYTE( x, y ) ( ( (y) << 4 ) | (x) )
+
+/**
+ * Define a row pair in a DES S-box
+ *
+ * @v x0..xf		Upper row of row pair
+ * @v y0..yf		Lower row of row pair
+ */
+#define SBOX( x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, xa, xb, xc, xd, xe, xf,  \
+	      y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, ya, yb, yc, yd, ye, yf ) \
+	SBYTE ( x0, y0 ), SBYTE ( x1, y1 ), SBYTE ( x2, y2 ), SBYTE ( x3, y3 ),\
+	SBYTE ( x4, y4 ), SBYTE ( x5, y5 ), SBYTE ( x6, y6 ), SBYTE ( x7, y7 ),\
+	SBYTE ( x8, y8 ), SBYTE ( x9, y9 ), SBYTE ( xa, ya ), SBYTE ( xb, yb ),\
+	SBYTE ( xc, yc ), SBYTE ( xd, yd ), SBYTE ( xe, ye ), SBYTE ( xf, yf )
+
+/** DES S-boxes S1..S8 */
+static const uint8_t des_s[8][32] = { {
+	/* S1 */
+	SBOX ( 14,  4, 13,  1,  2, 15, 11,  8,  3, 10,  6, 12,  5,  9,  0,  7,
+		0, 15,  7,  4, 14,  2, 13,  1, 10,  6, 12, 11,  9,  5,  3,  8 ),
+	SBOX (  4,  1, 14,  8, 13,  6,  2, 11, 15, 12,  9,  7,  3, 10,  5,  0,
+	       15, 12,  8,  2,  4,  9,  1,  7,  5, 11,  3, 14, 10,  0,  6, 13 )
+}, {
+	/* S2 */
+	SBOX ( 15,  1,  8, 14,  6, 11,  3,  4,  9,  7,  2, 13, 12,  0,  5, 10,
+		3, 13,  4,  7, 15,  2,  8, 14, 12,  0,  1, 10,  6,  9, 11,  5 ),
+	SBOX (  0, 14,  7, 11, 10,  4, 13,  1,  5,  8, 12,  6,  9,  3,  2, 15,
+		13,  8, 10,  1,  3, 15,  4,  2, 11,  6,  7, 12,  0,  5, 14,  9 )
+}, {
+	/* S3 */
+	SBOX ( 10,  0,  9, 14,  6,  3, 15,  5,  1, 13, 12,  7, 11,  4,  2,  8,
+	       13,  7,  0,  9,  3,  4,  6, 10,  2,  8,  5, 14, 12, 11, 15,  1 ),
+	SBOX ( 13,  6,  4,  9,  8, 15,  3,  0, 11,  1,  2, 12,  5, 10, 14,  7,
+		1, 10, 13,  0,  6,  9,  8,  7,  4, 15, 14,  3, 11,  5,  2, 12 )
+}, {
+	/* S4 */
+	SBOX (  7, 13, 14,  3,  0,  6,  9, 10,  1,  2,  8,  5, 11, 12,  4, 15,
+	       13,  8, 11,  5,  6, 15,  0,  3,  4,  7,  2, 12,  1, 10, 14,  9 ),
+	SBOX ( 10,  6,  9,  0, 12, 11,  7, 13, 15,  1,  3, 14,  5,  2,  8,  4,
+		3, 15,  0,  6, 10,  1, 13,  8,  9,  4,  5, 11, 12,  7,  2, 14 )
+}, {
+	/* S5 */
+	SBOX (  2, 12,  4,  1,  7, 10, 11,  6,  8,  5,  3, 15, 13,  0, 14,  9,
+	       14, 11,  2, 12,  4,  7, 13,  1,  5,  0, 15, 10,  3,  9,  8,  6 ),
+	SBOX (  4,  2,  1, 11, 10, 13,  7,  8, 15,  9, 12,  5,  6,  3,  0, 14,
+	       11,  8, 12,  7,  1, 14,  2, 13,  6, 15,  0,  9, 10,  4,  5,  3 )
+}, {
+	/* S6 */
+	SBOX ( 12,  1, 10, 15,  9,  2,  6,  8,  0, 13,  3,  4, 14,  7,  5, 11,
+	       10, 15,  4,  2,  7, 12,  9,  5,  6,  1, 13, 14,  0, 11,  3,  8 ),
+	SBOX (  9, 14, 15,  5,  2,  8, 12,  3,  7,  0,  4, 10,  1, 13, 11,  6,
+		4,  3,  2, 12,  9,  5, 15, 10, 11, 14,  1,  7,  6,  0,  8, 13 )
+}, {
+	/* S7 */
+	SBOX (  4, 11,  2, 14, 15,  0,  8, 13,  3, 12,  9,  7,  5, 10,  6,  1,
+	       13,  0, 11,  7,  4,  9,  1, 10, 14,  3,  5, 12,  2, 15,  8,  6 ),
+	SBOX (  1,  4, 11, 13, 12,  3,  7, 14, 10, 15,  6,  8,  0,  5,  9,  2,
+		6, 11, 13,  8,  1,  4, 10,  7,  9,  5,  0, 15, 14,  2,  3, 12 )
+}, {
+	/* S8 */
+	SBOX ( 13,  2,  8,  4,  6, 15, 11,  1, 10,  9,  3, 14,  5,  0, 12,  7,
+		1, 15, 13,  8, 10,  3,  7,  4, 12,  5,  6, 11,  0, 14,  9,  2 ),
+	SBOX (  7, 11,  4,  1,  9, 12, 14,  2,  0,  6, 10, 13, 15,  3,  5,  8,
+		2,  1, 14,  7,  4, 10,  8, 13, 15, 12,  9,  0,  3,  5,  6, 11 )
+} };
+
+/**
+ * Define a bit index within permuted choice 2 (PC2)
+ *
+ * @v bit		Bit index
+ *
+ * Permuted choice 2 (PC2) is used to select bits from a concatenated
+ * pair of 28-bit registers ("C" and "D") as part of the key schedule.
+ * We store these as 32-bit registers and so must add 4 to indexes
+ * above 28.
+ */
+#define DES_PC2( x ) ( (x) + ( ( (x) > 28 ) ? 4 : 0 ) )
+
+/**
+ * Define six bits of permuted choice 2 (PC2)
+ *
+ * @v r1:r0		Bits corresponding to S-box row index
+ * @v c3:c0		Bits corresponding to S-box column index
+ *
+ * There are 8 steps within a DES round (one step per S-box).  Each
+ * step requires six bits of the round key, corresponding to the S-box
+ * input value {r1,c3,c2,c1,c0,r0}, where {r1,r0} is the table row
+ * index and {c3,c2,c1,c0} is the table column index.
+ *
+ * As an optimisation, we store the least significant of the 6 bits in
+ * the sign bit of a signed 8-bit value, and the remaining 5 bits in
+ * the least significant 5 bits of the 8-bit value.  See the comments
+ * in des_sbox() for further details.
+ */
+#define DES_PC2R( r1, c3, c2, c1, c0, r0 )				\
+	DES_PC2 ( r0 ), /* LSB stored in sign bit */			\
+	DES_PC2 ( r0 ), /* Unused bit */				\
+	DES_PC2 ( r0 ), /* Unused bit */				\
+	DES_PC2 ( r1 ),	/* Remaining 5 bits */				\
+	DES_PC2 ( c3 ),	/* ... */					\
+	DES_PC2 ( c2 ),	/* ... */					\
+	DES_PC2 ( c1 ),	/* ... */					\
+	DES_PC2 ( c0 ) 	/* ... */
+
+/**
+ * A DES systematic permutation generator
+ *
+ * Many of the permutations used in DES comprise systematic bit
+ * patterns.  We generate these permutations at runtime to save on
+ * code size.
+ */
+struct des_generator {
+	/** Permutation */
+	uint8_t *permutation;
+	/** Seed value */
+	uint32_t seed;
+};
+
+/**
+ * Define a DES permutation generator
+ *
+ * @v PERMUTATION	Permutation
+ * @v OFFSET		Fixed input bit offset (0 or 1)
+ * @v INV<n>		Input bit index bit <n> should be inverted
+ * @v BIT<n>		Source bit for input bit index bit <n>
+ * @ret generator	Permutation generator
+ */
+#define DES_GENERATOR( PERMUTATION, OFFSET, INV5, BIT5, INV4, BIT4,	\
+		       INV3, BIT3, INV2, BIT2, INV1, BIT1, INV0, BIT0 )	\
+	{								\
+		.permutation = (PERMUTATION),				\
+		.seed = ( ( (INV0) << 31 ) | ( (BIT0) << 28 ) |		\
+			  ( (INV1) << 27 ) | ( (BIT1) << 24 ) |		\
+			  ( (INV2) << 23 ) | ( (BIT2) << 20 ) |		\
+			  ( (INV3) << 19 ) | ( (BIT3) << 16 ) |		\
+			  ( (INV4) << 15 ) | ( (BIT4) << 12 ) |		\
+			  ( (INV5) << 11 ) | ( (BIT5) <<  8 ) |		\
+			  ( ( uint32_t ) sizeof (PERMUTATION) - 1 ) |	\
+			  (OFFSET) ),					\
+	}
+
+/** DES permuted choice 1 (PC1) "C" register */
+static uint8_t des_pc1c[29];
+
+/** DES permuted choice 1 (PC1) "D" register */
+static uint8_t des_pc1d[33];
+
+/** DES permuted choice 2 (PC2) */
+static const uint8_t des_pc2[65] = {
+	DES_PC2R ( 14, 17, 11, 24,  1,  5 ),
+	DES_PC2R (  3, 28, 15,  6, 21, 10 ),
+	DES_PC2R ( 23, 19, 12,  4, 26,  8 ),
+	DES_PC2R ( 16,  7, 27, 20, 13,  2 ),
+	DES_PC2R ( 41, 52, 31, 37, 47, 55 ),
+	DES_PC2R ( 30, 40, 51, 45, 33, 48 ),
+	DES_PC2R ( 44, 49, 39, 56, 34, 53 ),
+	DES_PC2R ( 46, 42, 50, 36, 29, 32 ),
+	0 /* terminator */
+};
+
+/** DES initial permutation (IP) */
+static uint8_t des_ip[65];
+
+/** DES data permutation (P) */
+static const uint8_t des_p[33] = {
+	16,  7, 20, 21, 29, 12, 28, 17,  1, 15, 23, 26,  5, 18, 31, 10,
+	 2,  8, 24, 14, 32, 27,  3,  9, 19, 13, 30,  6, 22, 11,  4, 25,
+	 0 /* terminator */
+};
+
+/** DES final / inverse initial permutation (FP / IP^-1) */
+static uint8_t des_fp[65];
+
+/** DES permutation generators */
+static struct des_generator des_generators[] = {
+
+	/* The DES initial permutation transforms the bit index
+	 * {x5,x4,x3,x2,x1,x0}+1 into {~x2,~x1,~x0,x4,x3,~x5}+1
+	 */
+	DES_GENERATOR ( des_ip, 1, 1, 2, 1, 1, 1, 0, 0, 4, 0, 3, 1, 5 ),
+
+	/* The DES final permutation transforms the bit index
+	 * {x5,x4,x3,x2,x1,x0}+1 into {~x0,x2,x1,~x5,~x4,~x3}+1
+	 *
+	 * There is an asymmetry in the DES block diagram for the last
+	 * of the 16 rounds, which is functionally equivalent to
+	 * performing 16 identical rounds and then swapping the left
+	 * and right halves before applying the final permutation.  We
+	 * may therefore account for this asymmetry by inverting the
+	 * MSB in each bit index, to point to the corresponding bit in
+	 * the other half.
+	 *
+	 * This is equivalent to using a permutation that transforms
+	 * {x5,x4,x3,x2,x1,x0}+1 into {x0,x2,x1,~x5,~x4,~x3}+1
+	 */
+	DES_GENERATOR ( des_fp, 1, 0, 0, 0, 2, 0, 1, 1, 5, 1, 4, 1, 3 ),
+
+	/* The "C" half of DES permuted choice 1 (PC1) transforms the
+	 * bit index {x5,x4,x3,x2,x1,x0}+1 into {~x2,~x1,~x0,x5,x4,x3}+1
+	 */
+	DES_GENERATOR ( des_pc1c, 1, 1, 2, 1, 1, 1, 0, 0, 5, 0, 4, 0, 3 ),
+
+	/* The "D" half of DES permuted choice 1 (PC1) transforms the
+	 * bit index {x5,x4,x3,x2,x1,x0}+1 into {~x2,~x1,~x0,~x5,~x4,~x3}+0
+	 *
+	 * Due to the idosyncratic design choice of using 28-bit
+	 * registers in the DES key expansion schedule, the final four
+	 * permutation values appear at indices [28:31] instead of
+	 * [24:27].  This is adjusted for in @c des_setkey().
+	 */
+	DES_GENERATOR ( des_pc1d, 0, 1, 2, 1, 1, 1, 0, 1, 5, 1, 4, 1, 3 ),
+};
+
+/**
+ * Generate DES permutation
+ *
+ * @v generator		Generator
+ */
+static __attribute__ (( noinline )) void
+des_generate ( struct des_generator *generator ) {
+	uint8_t *permutation = generator->permutation;
+	uint32_t seed = generator->seed;
+	unsigned int index = 0;
+	uint8_t accum;
+	uint8_t bit;
+
+	/* Generate permutations
+	 *
+	 * This loop is optimised for code size on a
+	 * register-constrained architecture such as i386.
+	 */
+	do {
+		/* Rotate seed to access MSB's bit descriptor */
+		seed = ror32 ( seed, 8 );
+
+		/* Initialise accumulator with six flag bits */
+		accum = 0xfc;
+
+		/* Accumulate bits until all six flag bits are cleared */
+		do {
+			/* Extract specified bit from index.  Use a
+			 * rotation instead of a shift, since this
+			 * will allow the mask to be elided.
+			 */
+			bit = ror8 ( index, ( seed & 0x07 ) );
+			seed = ror32 ( seed, 3 );
+
+			/* Toggle bit if applicable */
+			bit ^= seed;
+			seed = ror32 ( seed, 1 );
+
+			/* Add bit to accumulator and clear one flag bit */
+			accum <<= 1;
+			accum |= ( bit & 0x01 );
+
+		} while ( accum & 0x80 );
+
+		/* Add constant offset if applicable */
+		accum += ( seed & 0x01 );
+
+		/* Store permutation */
+		permutation[index] = accum;
+
+		/* Loop until reaching length (which is always even) */
+	} while ( ++index < ( seed & 0xfe ) );
+	DBGC2 ( permutation, "DES generated permutation %p:\n", permutation );
+	DBGC2_HDA ( permutation, 0, permutation,
+		    ( ( seed & 0xfe ) + 1 /* zero terminator */ ) );
+}
+
+/**
+ * Initialise permutations
+ */
+static void des_init ( void ) {
+	unsigned int i;
+
+	/* Generate all generated permutations */
+	for ( i = 0 ; i < ( sizeof ( des_generators ) /
+			    sizeof ( des_generators[0] ) ) ; i++ ) {
+		des_generate ( &des_generators[i] );
+	}
+}
+
+/** Initialisation function */
+struct init_fn des_init_fn __init_fn ( INIT_NORMAL ) = {
+	.initialise = des_init,
+};
+
+/**
+ * Perform bit permutation
+ *
+ * @v permutation	Bit permutation (zero-terminated)
+ * @v in		Input value
+ * @v out		Output value
+ */
+static void des_permute ( const uint8_t *permutation, const uint8_t *in,
+			  uint8_t *out ) {
+	uint8_t mask = 0x80;
+	uint8_t accum = 0;
+	unsigned int bit;
+
+	/* Extract individual input bits to construct output value */
+	while ( ( bit = *(permutation++) ) ) {
+		bit--;
+		if ( in[ bit / 8 ] & ( 0x80 >> ( bit % 8 ) ) )
+			accum |= mask;
+		*out = accum;
+		mask = ror8 ( mask, 1 );
+		if ( mask == 0x80 ) {
+			out++;
+			accum = 0;
+		}
+	}
+}
+
+/**
+ * Perform DES S-box substitution
+ *
+ * @v in		32-bit input value (native endian)
+ * @v rkey		48-bit round key
+ * @ret out		32-bit output value (native endian)
+ */
+static uint32_t des_sbox ( uint32_t in, const union des_round_key *rkey ) {
+	uint32_t out = 0;
+	uint32_t lookup;
+	int32_t key;
+	uint8_t sub;
+	unsigned int i;
+
+	/* Perform input expansion, key addition, and S-box substitution */
+	for ( i = 0 ; i < 8 ; i++ ) {
+
+		/* Rotate input and output */
+		out = rol32 ( out, 4 );
+		in = rol32 ( in, 4 );
+
+		/* Extract step key from relevant 6 bits of round key
+		 *
+		 * The least significant of the 6 bits (corresponding
+		 * to bit r0 in the S-box lookup index) is stored in
+		 * the sign bit of the step key byte.  It will
+		 * therefore be propagated via sign extension to the
+		 * MSB of the 32-bit step key.
+		 *
+		 * The remaining 5 of the 6 bits (corresponding to
+		 * bits {r1,c3,c2,c1,c0} in the S-box lookup index)
+		 * are stored in the least significant 5 bits of the
+		 * step key byte and will end up in the least
+		 * significant 5 bits of the 32-bit step key.
+		 */
+		key = rkey->step[i];
+
+		/* Add step key to input to produce S-box lookup index
+		 *
+		 * We do not ever perform an explicit expansion of the
+		 * input value from 32 to 48 bits.  Instead, we rotate
+		 * the 32-bit input value by 4 bits on each step, and
+		 * extract the relevant 6 bits.
+		 *
+		 * The least significant of the 6 bits (corresponding
+		 * to bit r0 in the S-box lookup index) is currently
+		 * in the MSB of the 32-bit (rotated) input value.
+		 *
+		 * The remaining 5 of the 6 bits (corresponding to
+		 * bits {r1,c3,c2,c1,c0} in the S-box lookup index)
+		 * are currently in the least significant 5 bits of
+		 * the 32-bit (rotated) input value.
+		 *
+		 * This aligns with the placement of the bits in the
+		 * step key (see above), and we can therefore perform
+		 * a single XOR to add the 6-bit step key to the
+		 * relevant 6 bits of the input value.
+		 */
+		lookup = ( in ^ key );
+
+		/* Look up S[i][in ^ key] from S-box
+		 *
+		 * We have bits {r1,c3,c2,c1,c0} in the least
+		 * significant 5 bits of the lookup index, and so can
+		 * use the masked lookup index directly as a byte
+		 * index into the relevant S-box to extract the byte
+		 * containing both {r1,c3,c2,c1,c0,'0'} and
+		 * {r1,c3,c2,c1,c0,'1'}.
+		 *
+		 * We then use the MSB of the 32-bit lookup index to
+		 * extract the relevant nibble for the full lookup
+		 * index {r1,c3,c2,c1,c0,r0}.
+		 */
+		sub = des_s[i][ lookup & 0x1f ];
+		sub >>= ( ( lookup >> 29 ) & 4 );
+		sub &= 0x0f;
+
+		/* Substitute S[i][input ^ key] into output */
+		out |= sub;
+	}
+
+	return out;
+}
+
+/**
+ * Perform a single DES round
+ *
+ * @v block		DES block
+ * @v rkey		48-bit round key
+ */
+static void des_round ( union des_block *block,
+			const union des_round_key *rkey ) {
+	union des_dword sbox;
+	uint32_t left;
+	uint32_t right;
+
+	/* Extract left and right halves L[n-1] and R[n-1] */
+	left = block->left.dword;
+	right = block->right.dword;
+	DBGC2 ( block, "DES L=%08x R=%08x K=%08x%08x", be32_to_cpu ( left ),
+		be32_to_cpu ( right ), be32_to_cpu ( rkey->dword[0] ),
+		be32_to_cpu ( rkey->dword[1] ) );
+
+	/* L[n] = R[n-1] */
+	block->left.dword = right;
+
+	/* Calculate Feistel function f(R[n-1], K[n]) */
+	sbox.dword = cpu_to_be32 ( des_sbox ( be32_to_cpu ( right ), rkey ) );
+	des_permute ( des_p, sbox.byte, block->right.byte );
+
+	/* R[n] = L[n-1] + f(R[n-1], K[n]) */
+	block->right.dword ^= left;
+	DBGC2 ( block, " => L=%08x R=%08x\n",
+		be32_to_cpu ( block->left.dword ),
+		be32_to_cpu ( block->right.dword ) );
+}
+
+/**
+ * Perform all DES rounds
+ *
+ * @v in		Input DES block
+ * @v out		Output DES block
+ * @v rkey		Starting 48-bit round key
+ * @v offset		Byte offset between round keys
+ */
+static void des_rounds ( const union des_block *in, union des_block *out,
+			 const union des_round_key *rkey,
+			 ssize_t offset ) {
+	union des_block tmp;
+	unsigned int i;
+
+	/* Apply initial permutation */
+	des_permute ( des_ip, in->byte, tmp.byte );
+
+	/* Perform all DES rounds, consuming keys in the specified order */
+	for ( i = 0 ; i < DES_ROUNDS ; i++ ) {
+		des_round ( &tmp, rkey );
+		rkey = ( ( ( void * ) rkey ) + offset );
+	}
+
+	/* Apply final permutation */
+	DBGC ( &tmp, "DES %scrypted %08x%08x => ",
+	       ( ( offset > 0 ) ? "en" : "de" ), be32_to_cpu ( in->dword[0] ),
+	       be32_to_cpu ( in->dword[1] ) );
+	des_permute ( des_fp, tmp.byte, out->byte );
+	DBGC ( &tmp, "%08x%08x\n", be32_to_cpu ( out->dword[0] ),
+	       be32_to_cpu ( out->dword[1] ) );
+}
+
+/**
+ * Rotate 28-bit word
+ *
+ * @v dword		28-bit dword value
+ * @ret dword		Rotated 28-bit dword value
+ */
+static uint32_t des_rol28 ( uint32_t dword ) {
+	int32_t sdword;
+
+	/* Convert to native-endian */
+	sdword = be32_to_cpu ( dword );
+
+	/* Signed shift right by 4 places to copy bit 31 to bits 27:31 */
+	sdword >>= 4;
+
+	/* Rotate left */
+	sdword = rol32 ( sdword, 1 );
+
+	/* Shift left by 4 places to restore bit positions */
+	sdword <<= 4;
+
+	/* Convert back to big-endian */
+	dword = cpu_to_be32 ( sdword );
+
+	return dword;
+}
+
+/**
+ * Set key
+ *
+ * @v ctx		Context
+ * @v key		Key
+ * @v keylen		Key length
+ * @ret rc		Return status code
+ */
+static int des_setkey ( void *ctx, const void *key, size_t keylen ) {
+	struct des_context *des = ctx;
+	union des_round_key *rkey = des->rkey;
+	union des_block reg;
+	uint32_t schedule;
+
+	/* Validate key length */
+	if ( keylen != DES_BLOCKSIZE )
+		return -EINVAL;
+	DBGC ( des, "DES %p new key:\n", des );
+	DBGC_HDA ( des, 0, key, keylen );
+
+	/* Apply permuted choice 1 */
+	des_permute ( des_pc1c, key, reg.c.byte );
+	des_permute ( des_pc1d, key, reg.d.byte );
+	reg.d.byte[3] <<= 4; /* see comment for @c des_pc1d */
+	DBGC2 ( des, "DES %p C[ 0]=%07x D[ 0]=%07x\n",
+		des, ( be32_to_cpu ( reg.c.dword ) >> 4 ),
+		( be32_to_cpu ( reg.d.dword ) >> 4 ) );
+
+	/* Generate round keys */
+	for ( schedule = DES_SCHEDULE ; schedule ; schedule >>= 1 ) {
+
+		/* Shift 28-bit words */
+		reg.c.dword = des_rol28 ( reg.c.dword );
+		reg.d.dword = des_rol28 ( reg.d.dword );
+
+		/* Skip rounds according to shift schedule */
+		if ( ! ( schedule & 1 ) )
+			continue;
+
+		/* Apply permuted choice 2 */
+		des_permute ( des_pc2, reg.byte, rkey->byte );
+		DBGC2 ( des, "DES %p C[%2zd]=%07x D[%2zd]=%07x K[%2zd]="
+			"%08x%08x\n", des, ( ( rkey - des->rkey ) + 1 ),
+			( be32_to_cpu ( reg.c.dword ) >> 4 ),
+			( ( rkey - des->rkey ) + 1 ),
+			( be32_to_cpu ( reg.d.dword ) >> 4 ),
+			( ( rkey - des->rkey ) + 1 ),
+			be32_to_cpu ( rkey->dword[0] ),
+			be32_to_cpu ( rkey->dword[1] ) );
+
+		/* Move to next key */
+		rkey++;
+	}
+
+	/* Sanity check */
+	assert ( rkey == &des->rkey[DES_ROUNDS] );
+
+	return 0;
+}
+
+/**
+ * Encrypt data
+ *
+ * @v ctx		Context
+ * @v src		Data to encrypt
+ * @v dst		Buffer for encrypted data
+ * @v len		Length of data
+ */
+static void des_encrypt ( void *ctx, const void *src, void *dst, size_t len ) {
+	struct des_context *des = ctx;
+
+	/* Sanity check */
+	assert ( len == DES_BLOCKSIZE );
+
+	/* Cipher using keys in forward direction */
+	des_rounds ( src, dst, &des->rkey[0], sizeof ( des->rkey[0] ) );
+}
+
+/**
+ * Decrypt data
+ *
+ * @v ctx		Context
+ * @v src		Data to decrypt
+ * @v dst		Buffer for decrypted data
+ * @v len		Length of data
+ */
+static void des_decrypt ( void *ctx, const void *src, void *dst, size_t len ) {
+	struct des_context *des = ctx;
+
+	/* Sanity check */
+	assert ( len == DES_BLOCKSIZE );
+
+	/* Cipher using keys in reverse direction */
+	des_rounds ( src, dst, &des->rkey[ DES_ROUNDS - 1 ],
+		     -sizeof ( des->rkey[0] ) );
+}
+
+/** Basic DES algorithm */
+struct cipher_algorithm des_algorithm = {
+	.name = "des",
+	.ctxsize = sizeof ( struct des_context ),
+	.blocksize = DES_BLOCKSIZE,
+	.alignsize = 0,
+	.authsize = 0,
+	.setkey = des_setkey,
+	.setiv = cipher_null_setiv,
+	.encrypt = des_encrypt,
+	.decrypt = des_decrypt,
+	.auth = cipher_null_auth,
+};
+
+/* DES in Electronic Codebook mode */
+ECB_CIPHER ( des_ecb, des_ecb_algorithm,
+	     des_algorithm, struct des_context, DES_BLOCKSIZE );
+
+/* DES in Cipher Block Chaining mode */
+CBC_CIPHER ( des_cbc, des_cbc_algorithm,
+	     des_algorithm, struct des_context, DES_BLOCKSIZE );
diff --git a/src/crypto/gcm.c b/src/crypto/gcm.c
index 9d8bae82..a32890d5 100644
--- a/src/crypto/gcm.c
+++ b/src/crypto/gcm.c
@@ -469,13 +469,15 @@ int gcm_setkey ( struct gcm_context *context, const void *key, size_t keylen,
  * @v ivlen		Initialisation vector length
  */
 void gcm_setiv ( struct gcm_context *context, const void *iv, size_t ivlen ) {
-	union gcm_block *check = ( ( void * ) context );
 
-	/* Sanity checks */
-	linker_assert ( &context->hash == check, gcm_bad_layout );
-	linker_assert ( &context->len == check + 1, gcm_bad_layout );
-	linker_assert ( &context->ctr == check + 2, gcm_bad_layout );
-	linker_assert ( &context->key == check + 3, gcm_bad_layout );
+	/* Sanity check: ensure that memset()s will clear expected state */
+	build_assert ( &context->hash < &context->ctr );
+	build_assert ( &context->len < &context->ctr );
+	build_assert ( &context->ctr < &context->key );
+	build_assert ( ( ( void * ) &context->raw_cipher ) >
+		       ( ( void * ) &context->key ) );
+	build_assert ( ( ( void * ) context->raw_ctx ) >
+		       ( ( void * ) &context->key ) );
 
 	/* Reset non-key state */
 	memset ( context, 0, offsetof ( typeof ( *context ), key ) );
diff --git a/src/crypto/md4.c b/src/crypto/md4.c
index ca5dcc21..dcd86a42 100644
--- a/src/crypto/md4.c
+++ b/src/crypto/md4.c
@@ -155,11 +155,11 @@ static void md4_digest ( struct md4_context *context ) {
 
 	/* Sanity checks */
 	assert ( ( context->len % sizeof ( context->ddd.dd.data ) ) == 0 );
-	linker_assert ( &u.ddd.dd.digest.h[0] == a, md4_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[1] == b, md4_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[2] == c, md4_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[3] == d, md4_bad_layout );
-	linker_assert ( &u.ddd.dd.data.dword[0] == w, md4_bad_layout );
+	build_assert ( &u.ddd.dd.digest.h[0] == a );
+	build_assert ( &u.ddd.dd.digest.h[1] == b );
+	build_assert ( &u.ddd.dd.digest.h[2] == c );
+	build_assert ( &u.ddd.dd.digest.h[3] == d );
+	build_assert ( &u.ddd.dd.data.dword[0] == w );
 
 	DBGC ( context, "MD4 digesting:\n" );
 	DBGC_HDA ( context, 0, &context->ddd.dd.digest,
diff --git a/src/crypto/md5.c b/src/crypto/md5.c
index bee382e9..5c62513e 100644
--- a/src/crypto/md5.c
+++ b/src/crypto/md5.c
@@ -178,11 +178,11 @@ static void md5_digest ( struct md5_context *context ) {
 
 	/* Sanity checks */
 	assert ( ( context->len % sizeof ( context->ddd.dd.data ) ) == 0 );
-	linker_assert ( &u.ddd.dd.digest.h[0] == a, md5_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[1] == b, md5_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[2] == c, md5_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[3] == d, md5_bad_layout );
-	linker_assert ( &u.ddd.dd.data.dword[0] == w, md5_bad_layout );
+	build_assert ( &u.ddd.dd.digest.h[0] == a );
+	build_assert ( &u.ddd.dd.digest.h[1] == b );
+	build_assert ( &u.ddd.dd.digest.h[2] == c );
+	build_assert ( &u.ddd.dd.digest.h[3] == d );
+	build_assert ( &u.ddd.dd.data.dword[0] == w );
 
 	DBGC ( context, "MD5 digesting:\n" );
 	DBGC_HDA ( context, 0, &context->ddd.dd.digest,
diff --git a/src/crypto/mishmash/dhe_rsa_aes_cbc_sha1.c b/src/crypto/mishmash/dhe_rsa_aes_cbc_sha1.c
new file mode 100644
index 00000000..05e409f7
--- /dev/null
+++ b/src/crypto/mishmash/dhe_rsa_aes_cbc_sha1.c
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/rsa.h>
+#include <ipxe/aes.h>
+#include <ipxe/sha1.h>
+#include <ipxe/sha256.h>
+#include <ipxe/tls.h>
+
+/** TLS_DHE_RSA_WITH_AES_128_CBC_SHA cipher suite */
+struct tls_cipher_suite
+tls_dhe_rsa_with_aes_128_cbc_sha __tls_cipher_suite ( 15 ) = {
+	.code = htons ( TLS_DHE_RSA_WITH_AES_128_CBC_SHA ),
+	.key_len = ( 128 / 8 ),
+	.fixed_iv_len = 0,
+	.record_iv_len = AES_BLOCKSIZE,
+	.mac_len = SHA1_DIGEST_SIZE,
+	.exchange = &tls_dhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha1_algorithm,
+	.handshake = &sha256_algorithm,
+};
+
+/** TLS_DHE_RSA_WITH_AES_256_CBC_SHA cipher suite */
+struct tls_cipher_suite
+tls_dhe_rsa_with_aes_256_cbc_sha __tls_cipher_suite ( 16 ) = {
+	.code = htons ( TLS_DHE_RSA_WITH_AES_256_CBC_SHA ),
+	.key_len = ( 256 / 8 ),
+	.fixed_iv_len = 0,
+	.record_iv_len = AES_BLOCKSIZE,
+	.mac_len = SHA1_DIGEST_SIZE,
+	.exchange = &tls_dhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha1_algorithm,
+	.handshake = &sha256_algorithm,
+};
diff --git a/src/crypto/mishmash/dhe_rsa_aes_cbc_sha256.c b/src/crypto/mishmash/dhe_rsa_aes_cbc_sha256.c
new file mode 100644
index 00000000..6ce42864
--- /dev/null
+++ b/src/crypto/mishmash/dhe_rsa_aes_cbc_sha256.c
@@ -0,0 +1,60 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/rsa.h>
+#include <ipxe/aes.h>
+#include <ipxe/sha256.h>
+#include <ipxe/tls.h>
+
+/** TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 cipher suite */
+struct tls_cipher_suite
+tls_dhe_rsa_with_aes_128_cbc_sha256 __tls_cipher_suite ( 13 ) = {
+	.code = htons ( TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 ),
+	.key_len = ( 128 / 8 ),
+	.fixed_iv_len = 0,
+	.record_iv_len = AES_BLOCKSIZE,
+	.mac_len = SHA256_DIGEST_SIZE,
+	.exchange = &tls_dhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha256_algorithm,
+	.handshake = &sha256_algorithm,
+};
+
+/** TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 cipher suite */
+struct tls_cipher_suite
+tls_dhe_rsa_with_aes_256_cbc_sha256 __tls_cipher_suite ( 14 ) = {
+	.code = htons ( TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 ),
+	.key_len = ( 256 / 8 ),
+	.fixed_iv_len = 0,
+	.record_iv_len = AES_BLOCKSIZE,
+	.mac_len = SHA256_DIGEST_SIZE,
+	.exchange = &tls_dhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha256_algorithm,
+	.handshake = &sha256_algorithm,
+};
diff --git a/src/crypto/mishmash/dhe_rsa_aes_gcm_sha256.c b/src/crypto/mishmash/dhe_rsa_aes_gcm_sha256.c
new file mode 100644
index 00000000..dc5cad9f
--- /dev/null
+++ b/src/crypto/mishmash/dhe_rsa_aes_gcm_sha256.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2022 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/rsa.h>
+#include <ipxe/aes.h>
+#include <ipxe/sha256.h>
+#include <ipxe/tls.h>
+
+/** TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 cipher suite */
+struct tls_cipher_suite
+tls_dhe_rsa_with_aes_128_gcm_sha256 __tls_cipher_suite ( 11 ) = {
+	.code = htons ( TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 ),
+	.key_len = ( 128 / 8 ),
+	.fixed_iv_len = 4,
+	.record_iv_len = 8,
+	.mac_len = 0,
+	.exchange = &tls_dhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_gcm_algorithm,
+	.digest = &sha256_algorithm,
+	.handshake = &sha256_algorithm,
+};
diff --git a/src/crypto/mishmash/dhe_rsa_aes_gcm_sha384.c b/src/crypto/mishmash/dhe_rsa_aes_gcm_sha384.c
new file mode 100644
index 00000000..0448255f
--- /dev/null
+++ b/src/crypto/mishmash/dhe_rsa_aes_gcm_sha384.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2022 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/rsa.h>
+#include <ipxe/aes.h>
+#include <ipxe/sha512.h>
+#include <ipxe/tls.h>
+
+/** TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 cipher suite */
+struct tls_cipher_suite
+tls_dhe_rsa_with_aes_256_gcm_sha384 __tls_cipher_suite ( 12 ) = {
+	.code = htons ( TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 ),
+	.key_len = ( 256 / 8 ),
+	.fixed_iv_len = 4,
+	.record_iv_len = 8,
+	.mac_len = 0,
+	.exchange = &tls_dhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_gcm_algorithm,
+	.digest = &sha384_algorithm,
+	.handshake = &sha384_algorithm,
+};
diff --git a/src/crypto/mishmash/ecdhe_rsa_aes_cbc_sha1.c b/src/crypto/mishmash/ecdhe_rsa_aes_cbc_sha1.c
new file mode 100644
index 00000000..c23f65cc
--- /dev/null
+++ b/src/crypto/mishmash/ecdhe_rsa_aes_cbc_sha1.c
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/rsa.h>
+#include <ipxe/aes.h>
+#include <ipxe/sha1.h>
+#include <ipxe/sha256.h>
+#include <ipxe/tls.h>
+
+/** TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA cipher suite */
+struct tls_cipher_suite
+tls_ecdhe_rsa_with_aes_128_cbc_sha __tls_cipher_suite ( 05 ) = {
+	.code = htons ( TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA ),
+	.key_len = ( 128 / 8 ),
+	.fixed_iv_len = 0,
+	.record_iv_len = AES_BLOCKSIZE,
+	.mac_len = SHA1_DIGEST_SIZE,
+	.exchange = &tls_ecdhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha1_algorithm,
+	.handshake = &sha256_algorithm,
+};
+
+/** TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA cipher suite */
+struct tls_cipher_suite
+tls_ecdhe_rsa_with_aes_256_cbc_sha __tls_cipher_suite ( 06 ) = {
+	.code = htons ( TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA ),
+	.key_len = ( 256 / 8 ),
+	.fixed_iv_len = 0,
+	.record_iv_len = AES_BLOCKSIZE,
+	.mac_len = SHA1_DIGEST_SIZE,
+	.exchange = &tls_ecdhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha1_algorithm,
+	.handshake = &sha256_algorithm,
+};
diff --git a/src/crypto/mishmash/ecdhe_rsa_aes_cbc_sha256.c b/src/crypto/mishmash/ecdhe_rsa_aes_cbc_sha256.c
new file mode 100644
index 00000000..431e2e30
--- /dev/null
+++ b/src/crypto/mishmash/ecdhe_rsa_aes_cbc_sha256.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/rsa.h>
+#include <ipxe/aes.h>
+#include <ipxe/sha256.h>
+#include <ipxe/tls.h>
+
+/** TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 cipher suite */
+struct tls_cipher_suite
+tls_ecdhe_rsa_with_aes_128_cbc_sha256 __tls_cipher_suite ( 03 ) = {
+	.code = htons ( TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 ),
+	.key_len = ( 128 / 8 ),
+	.fixed_iv_len = 0,
+	.record_iv_len = AES_BLOCKSIZE,
+	.mac_len = SHA256_DIGEST_SIZE,
+	.exchange = &tls_ecdhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha256_algorithm,
+	.handshake = &sha256_algorithm,
+};
diff --git a/src/crypto/mishmash/ecdhe_rsa_aes_cbc_sha384.c b/src/crypto/mishmash/ecdhe_rsa_aes_cbc_sha384.c
new file mode 100644
index 00000000..c5297680
--- /dev/null
+++ b/src/crypto/mishmash/ecdhe_rsa_aes_cbc_sha384.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/rsa.h>
+#include <ipxe/aes.h>
+#include <ipxe/sha512.h>
+#include <ipxe/tls.h>
+
+/** TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 cipher suite */
+struct tls_cipher_suite
+tls_ecdhe_rsa_with_aes_256_cbc_sha384 __tls_cipher_suite ( 04 ) = {
+	.code = htons ( TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 ),
+	.key_len = ( 256 / 8 ),
+	.fixed_iv_len = 0,
+	.record_iv_len = AES_BLOCKSIZE,
+	.mac_len = SHA384_DIGEST_SIZE,
+	.exchange = &tls_ecdhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha384_algorithm,
+	.handshake = &sha384_algorithm,
+};
diff --git a/src/crypto/mishmash/ecdhe_rsa_aes_gcm_sha256.c b/src/crypto/mishmash/ecdhe_rsa_aes_gcm_sha256.c
new file mode 100644
index 00000000..4f4e38c6
--- /dev/null
+++ b/src/crypto/mishmash/ecdhe_rsa_aes_gcm_sha256.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/rsa.h>
+#include <ipxe/aes.h>
+#include <ipxe/sha256.h>
+#include <ipxe/tls.h>
+
+/** TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 cipher suite */
+struct tls_cipher_suite
+tls_ecdhe_rsa_with_aes_128_gcm_sha256 __tls_cipher_suite ( 01 ) = {
+	.code = htons ( TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 ),
+	.key_len = ( 128 / 8 ),
+	.fixed_iv_len = 4,
+	.record_iv_len = 8,
+	.mac_len = 0,
+	.exchange = &tls_ecdhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_gcm_algorithm,
+	.digest = &sha256_algorithm,
+	.handshake = &sha256_algorithm,
+};
diff --git a/src/crypto/mishmash/ecdhe_rsa_aes_gcm_sha384.c b/src/crypto/mishmash/ecdhe_rsa_aes_gcm_sha384.c
new file mode 100644
index 00000000..0bc7c305
--- /dev/null
+++ b/src/crypto/mishmash/ecdhe_rsa_aes_gcm_sha384.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/rsa.h>
+#include <ipxe/aes.h>
+#include <ipxe/sha512.h>
+#include <ipxe/tls.h>
+
+/** TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 cipher suite */
+struct tls_cipher_suite
+tls_ecdhe_rsa_with_aes_256_gcm_sha384 __tls_cipher_suite ( 02 ) = {
+	.code = htons ( TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 ),
+	.key_len = ( 256 / 8 ),
+	.fixed_iv_len = 4,
+	.record_iv_len = 8,
+	.mac_len = 0,
+	.exchange = &tls_ecdhe_exchange_algorithm,
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_gcm_algorithm,
+	.digest = &sha384_algorithm,
+	.handshake = &sha384_algorithm,
+};
diff --git a/src/crypto/mishmash/oid_x25519.c b/src/crypto/mishmash/oid_x25519.c
new file mode 100644
index 00000000..2f8aa065
--- /dev/null
+++ b/src/crypto/mishmash/oid_x25519.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/x25519.h>
+#include <ipxe/asn1.h>
+#include <ipxe/tls.h>
+
+/** "x25519" object identifier */
+static uint8_t oid_x25519[] = { ASN1_OID_X25519 };
+
+/** "x25519" OID-identified algorithm */
+struct asn1_algorithm x25519_algorithm __asn1_algorithm = {
+	.name = "x25519",
+	.curve = &x25519_curve,
+	.oid = ASN1_CURSOR ( oid_x25519 ),
+};
+
+/** X25519 named curve */
+struct tls_named_curve tls_x25519_named_curve __tls_named_curve ( 01 ) = {
+	.curve = &x25519_curve,
+	.code = htons ( TLS_NAMED_CURVE_X25519 ),
+};
diff --git a/src/crypto/mishmash/rsa_aes_cbc_sha1.c b/src/crypto/mishmash/rsa_aes_cbc_sha1.c
index 9f8193de..0862fb5a 100644
--- a/src/crypto/mishmash/rsa_aes_cbc_sha1.c
+++ b/src/crypto/mishmash/rsa_aes_cbc_sha1.c
@@ -30,39 +30,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/sha256.h>
 #include <ipxe/tls.h>
 
-/** TLS_DHE_RSA_WITH_AES_128_CBC_SHA cipher suite */
-struct tls_cipher_suite
-tls_dhe_rsa_with_aes_128_cbc_sha __tls_cipher_suite ( 05 ) = {
-	.code = htons ( TLS_DHE_RSA_WITH_AES_128_CBC_SHA ),
-	.key_len = ( 128 / 8 ),
-	.fixed_iv_len = 0,
-	.record_iv_len = AES_BLOCKSIZE,
-	.mac_len = SHA1_DIGEST_SIZE,
-	.exchange = &tls_dhe_exchange_algorithm,
-	.pubkey = &rsa_algorithm,
-	.cipher = &aes_cbc_algorithm,
-	.digest = &sha1_algorithm,
-	.handshake = &sha256_algorithm,
-};
-
-/** TLS_DHE_RSA_WITH_AES_256_CBC_SHA cipher suite */
-struct tls_cipher_suite
-tls_dhe_rsa_with_aes_256_cbc_sha __tls_cipher_suite ( 06 ) = {
-	.code = htons ( TLS_DHE_RSA_WITH_AES_256_CBC_SHA ),
-	.key_len = ( 256 / 8 ),
-	.fixed_iv_len = 0,
-	.record_iv_len = AES_BLOCKSIZE,
-	.mac_len = SHA1_DIGEST_SIZE,
-	.exchange = &tls_dhe_exchange_algorithm,
-	.pubkey = &rsa_algorithm,
-	.cipher = &aes_cbc_algorithm,
-	.digest = &sha1_algorithm,
-	.handshake = &sha256_algorithm,
-};
-
 /** TLS_RSA_WITH_AES_128_CBC_SHA cipher suite */
 struct tls_cipher_suite
-tls_rsa_with_aes_128_cbc_sha __tls_cipher_suite ( 15 ) = {
+tls_rsa_with_aes_128_cbc_sha __tls_cipher_suite ( 25 ) = {
 	.code = htons ( TLS_RSA_WITH_AES_128_CBC_SHA ),
 	.key_len = ( 128 / 8 ),
 	.fixed_iv_len = 0,
@@ -77,7 +47,7 @@ tls_rsa_with_aes_128_cbc_sha __tls_cipher_suite ( 15 ) = {
 
 /** TLS_RSA_WITH_AES_256_CBC_SHA cipher suite */
 struct tls_cipher_suite
-tls_rsa_with_aes_256_cbc_sha __tls_cipher_suite ( 16 ) = {
+tls_rsa_with_aes_256_cbc_sha __tls_cipher_suite ( 26 ) = {
 	.code = htons ( TLS_RSA_WITH_AES_256_CBC_SHA ),
 	.key_len = ( 256 / 8 ),
 	.fixed_iv_len = 0,
diff --git a/src/crypto/mishmash/rsa_aes_cbc_sha256.c b/src/crypto/mishmash/rsa_aes_cbc_sha256.c
index d0dc8496..e5928db8 100644
--- a/src/crypto/mishmash/rsa_aes_cbc_sha256.c
+++ b/src/crypto/mishmash/rsa_aes_cbc_sha256.c
@@ -29,39 +29,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/sha256.h>
 #include <ipxe/tls.h>
 
-/** TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 cipher suite */
-struct tls_cipher_suite
-tls_dhe_rsa_with_aes_128_cbc_sha256 __tls_cipher_suite ( 03 ) = {
-	.code = htons ( TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 ),
-	.key_len = ( 128 / 8 ),
-	.fixed_iv_len = 0,
-	.record_iv_len = AES_BLOCKSIZE,
-	.mac_len = SHA256_DIGEST_SIZE,
-	.exchange = &tls_dhe_exchange_algorithm,
-	.pubkey = &rsa_algorithm,
-	.cipher = &aes_cbc_algorithm,
-	.digest = &sha256_algorithm,
-	.handshake = &sha256_algorithm,
-};
-
-/** TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 cipher suite */
-struct tls_cipher_suite
-tls_dhe_rsa_with_aes_256_cbc_sha256 __tls_cipher_suite ( 04 ) = {
-	.code = htons ( TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 ),
-	.key_len = ( 256 / 8 ),
-	.fixed_iv_len = 0,
-	.record_iv_len = AES_BLOCKSIZE,
-	.mac_len = SHA256_DIGEST_SIZE,
-	.exchange = &tls_dhe_exchange_algorithm,
-	.pubkey = &rsa_algorithm,
-	.cipher = &aes_cbc_algorithm,
-	.digest = &sha256_algorithm,
-	.handshake = &sha256_algorithm,
-};
-
 /** TLS_RSA_WITH_AES_128_CBC_SHA256 cipher suite */
 struct tls_cipher_suite
-tls_rsa_with_aes_128_cbc_sha256 __tls_cipher_suite ( 13 ) = {
+tls_rsa_with_aes_128_cbc_sha256 __tls_cipher_suite ( 23 ) = {
 	.code = htons ( TLS_RSA_WITH_AES_128_CBC_SHA256 ),
 	.key_len = ( 128 / 8 ),
 	.fixed_iv_len = 0,
@@ -76,7 +46,7 @@ tls_rsa_with_aes_128_cbc_sha256 __tls_cipher_suite ( 13 ) = {
 
 /** TLS_RSA_WITH_AES_256_CBC_SHA256 cipher suite */
 struct tls_cipher_suite
-tls_rsa_with_aes_256_cbc_sha256 __tls_cipher_suite ( 14 ) = {
+tls_rsa_with_aes_256_cbc_sha256 __tls_cipher_suite ( 24 ) = {
 	.code = htons ( TLS_RSA_WITH_AES_256_CBC_SHA256 ),
 	.key_len = ( 256 / 8 ),
 	.fixed_iv_len = 0,
diff --git a/src/crypto/mishmash/rsa_aes_gcm_sha256.c b/src/crypto/mishmash/rsa_aes_gcm_sha256.c
index cf9c4c27..b18bbd84 100644
--- a/src/crypto/mishmash/rsa_aes_gcm_sha256.c
+++ b/src/crypto/mishmash/rsa_aes_gcm_sha256.c
@@ -29,24 +29,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/sha256.h>
 #include <ipxe/tls.h>
 
-/** TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 cipher suite */
-struct tls_cipher_suite
-tls_dhe_rsa_with_aes_128_gcm_sha256 __tls_cipher_suite ( 01 ) = {
-	.code = htons ( TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 ),
-	.key_len = ( 128 / 8 ),
-	.fixed_iv_len = 4,
-	.record_iv_len = 8,
-	.mac_len = 0,
-	.exchange = &tls_dhe_exchange_algorithm,
-	.pubkey = &rsa_algorithm,
-	.cipher = &aes_gcm_algorithm,
-	.digest = &sha256_algorithm,
-	.handshake = &sha256_algorithm,
-};
-
 /** TLS_RSA_WITH_AES_128_GCM_SHA256 cipher suite */
 struct tls_cipher_suite
-tls_rsa_with_aes_128_gcm_sha256 __tls_cipher_suite ( 11 ) = {
+tls_rsa_with_aes_128_gcm_sha256 __tls_cipher_suite ( 21 ) = {
 	.code = htons ( TLS_RSA_WITH_AES_128_GCM_SHA256 ),
 	.key_len = ( 128 / 8 ),
 	.fixed_iv_len = 4,
diff --git a/src/crypto/mishmash/rsa_aes_gcm_sha384.c b/src/crypto/mishmash/rsa_aes_gcm_sha384.c
index 10a977f7..06558aae 100644
--- a/src/crypto/mishmash/rsa_aes_gcm_sha384.c
+++ b/src/crypto/mishmash/rsa_aes_gcm_sha384.c
@@ -29,24 +29,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/sha512.h>
 #include <ipxe/tls.h>
 
-/** TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 cipher suite */
-struct tls_cipher_suite
-tls_dhe_rsa_with_aes_256_gcm_sha384 __tls_cipher_suite ( 02 ) = {
-	.code = htons ( TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 ),
-	.key_len = ( 256 / 8 ),
-	.fixed_iv_len = 4,
-	.record_iv_len = 8,
-	.mac_len = 0,
-	.exchange = &tls_dhe_exchange_algorithm,
-	.pubkey = &rsa_algorithm,
-	.cipher = &aes_gcm_algorithm,
-	.digest = &sha384_algorithm,
-	.handshake = &sha384_algorithm,
-};
-
 /** TLS_RSA_WITH_AES_256_GCM_SHA384 cipher suite */
 struct tls_cipher_suite
-tls_rsa_with_aes_256_gcm_sha384 __tls_cipher_suite ( 12 ) = {
+tls_rsa_with_aes_256_gcm_sha384 __tls_cipher_suite ( 22 ) = {
 	.code = htons ( TLS_RSA_WITH_AES_256_GCM_SHA384 ),
 	.key_len = ( 256 / 8 ),
 	.fixed_iv_len = 4,
diff --git a/src/crypto/mschapv2.c b/src/crypto/mschapv2.c
new file mode 100644
index 00000000..ac55fec1
--- /dev/null
+++ b/src/crypto/mschapv2.c
@@ -0,0 +1,363 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * MS-CHAPv2 authentication
+ *
+ * The algorithms used for MS-CHAPv2 authentication are defined in
+ * RFC 2759 section 8.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <byteswap.h>
+#include <ipxe/md4.h>
+#include <ipxe/sha1.h>
+#include <ipxe/des.h>
+#include <ipxe/mschapv2.h>
+
+/**
+ * MS-CHAPv2 context block
+ *
+ * For no particularly discernible reason, MS-CHAPv2 uses two
+ * different digest algorithms and one block cipher.  The uses do not
+ * overlap, so share the context storage between these to reduce stack
+ * usage.
+ */
+union mschapv2_context {
+	/** SHA-1 digest context */
+	uint8_t sha1[SHA1_CTX_SIZE];
+	/** MD4 digest context */
+	uint8_t md4[MD4_CTX_SIZE];
+	/** DES cipher context */
+	uint8_t des[DES_CTX_SIZE];
+};
+
+/**
+ * MS-CHAPv2 challenge hash
+ *
+ * MS-CHAPv2 calculates the SHA-1 digest of the peer challenge, the
+ * authenticator challenge, and the username, and then uses only the
+ * first 8 bytes of the result (as a DES plaintext block).
+ */
+union mschapv2_challenge_hash {
+	/** SHA-1 digest */
+	uint8_t sha1[SHA1_DIGEST_SIZE];
+	/** DES plaintext block */
+	uint8_t des[DES_BLOCKSIZE];
+};
+
+/**
+ * MS-CHAPv2 password hash
+ *
+ * MS-CHAPv2 calculates the MD4 digest of an unspecified two-byte
+ * little-endian Unicode encoding (presumably either UCS-2LE or
+ * UTF-16LE) of the password.
+ *
+ * For constructing the challenge response, the MD4 digest is then
+ * zero-padded to 21 bytes and used as three separate 56-bit DES keys.
+ *
+ * For constructing the authenticator response, the MD4 digest is then
+ * used as an input to a SHA-1 digest along with the NT response and a
+ * magic constant.
+ */
+union mschapv2_password_hash {
+	/** MD4 digest */
+	uint8_t md4[MD4_DIGEST_SIZE];
+	/** SHA-1 digest */
+	uint8_t sha1[SHA1_DIGEST_SIZE];
+	/** DES keys */
+	uint8_t des[3][DES_BLOCKSIZE];
+	/** DES key expansion */
+	uint8_t expand[ 3 * DES_BLOCKSIZE ];
+};
+
+/** MS-CHAPv2 magic constant 1 */
+static const char mschapv2_magic1[39] =
+	"Magic server to client signing constant";
+
+/** MS-CHAPv2 magic constant 2 */
+static const char mschapv2_magic2[41] =
+	"Pad to make it do more than one iteration";
+
+/**
+ * Calculate MS-CHAPv2 challenge hash
+ *
+ * @v ctx		Context block
+ * @v challenge		Authenticator challenge
+ * @v peer		Peer challenge
+ * @v username		User name (or NULL to use empty string)
+ * @v chash		Challenge hash to fill in
+ *
+ * This is the ChallengeHash() function as documented in RFC 2759
+ * section 8.2.
+ */
+static void
+mschapv2_challenge_hash ( union mschapv2_context *ctx,
+			  const struct mschapv2_challenge *challenge,
+			  const struct mschapv2_challenge *peer,
+			  const char *username,
+			  union mschapv2_challenge_hash *chash ) {
+	struct digest_algorithm *sha1 = &sha1_algorithm;
+
+	/* Calculate SHA-1 hash of challenges and username */
+	digest_init ( sha1, ctx->sha1 );
+	digest_update ( sha1, ctx->sha1, peer, sizeof ( *peer ) );
+	digest_update ( sha1, ctx->sha1, challenge, sizeof ( *challenge ) );
+	if ( username ) {
+		digest_update ( sha1, ctx->sha1, username,
+				strlen ( username ) );
+	}
+	digest_final ( sha1, ctx->sha1, chash->sha1 );
+	DBGC ( ctx, "MSCHAPv2 authenticator challenge:\n" );
+	DBGC_HDA ( ctx, 0, challenge, sizeof ( *challenge ) );
+	DBGC ( ctx, "MSCHAPv2 peer challenge:\n" );
+	DBGC_HDA ( ctx, 0, peer, sizeof ( *peer ) );
+	DBGC ( ctx, "MSCHAPv2 challenge hash:\n" );
+	DBGC_HDA ( ctx, 0, chash->des, sizeof ( chash->des ) );
+}
+
+/**
+ * Calculate MS-CHAPv2 password hash
+ *
+ * @v ctx		Context block
+ * @v password		Password (or NULL to use empty string)
+ * @v phash		Password hash to fill in
+ *
+ * This is the NtPasswordHash() function as documented in RFC 2759
+ * section 8.3.
+ */
+static void mschapv2_password_hash ( union mschapv2_context *ctx,
+				     const char *password,
+				     union mschapv2_password_hash *phash ) {
+	struct digest_algorithm *md4 = &md4_algorithm;
+	uint16_t wc;
+	uint8_t c;
+
+	/* Construct zero-padded MD4 hash of encoded password */
+	memset ( phash, 0, sizeof ( *phash ) );
+	digest_init ( md4, ctx->md4 );
+	if ( password ) {
+		while ( ( c = *(password++) ) ) {
+			wc = cpu_to_le16 ( c );
+			digest_update ( md4, ctx->md4, &wc, sizeof ( wc ) );
+		}
+	}
+	digest_final ( md4, ctx->md4, phash->md4 );
+	DBGC ( ctx, "MSCHAPv2 password hash:\n" );
+	DBGC_HDA ( ctx, 0, phash->md4, sizeof ( phash->md4 ) );
+}
+
+/**
+ * Hash the MS-CHAPv2 password hash
+ *
+ * @v ctx		Context block
+ * @v phash		Password hash to be rehashed
+ *
+ * This is the HashNtPasswordHash() function as documented in RFC 2759
+ * section 8.4.
+ */
+static void mschapv2_hash_hash ( union mschapv2_context *ctx,
+				 union mschapv2_password_hash *phash ) {
+	struct digest_algorithm *md4 = &md4_algorithm;
+
+	/* Calculate MD4 hash of existing MD4 hash */
+	digest_init ( md4, ctx->md4 );
+	digest_update ( md4, ctx->md4, phash->md4, sizeof ( phash->md4 ) );
+	digest_final ( md4, ctx->md4, phash->md4 );
+	DBGC ( ctx, "MSCHAPv2 password hash hash:\n" );
+	DBGC_HDA ( ctx, 0, phash->md4, sizeof ( phash->md4 ) );
+}
+
+/**
+ * Expand MS-CHAPv2 password hash by inserting DES dummy parity bits
+ *
+ * @v ctx		Context block
+ * @v phash		Password hash to expand
+ *
+ * This is part of the DesEncrypt() function as documented in RFC 2759
+ * section 8.6.
+ */
+static void mschapv2_expand_hash ( union mschapv2_context *ctx,
+				   union mschapv2_password_hash *phash ) {
+	uint8_t *dst;
+	uint8_t *src;
+	unsigned int i;
+
+	/* Expand password hash by inserting (unused) DES parity bits */
+	for ( i = ( sizeof ( phash->expand ) - 1 ) ; i > 0 ; i-- ) {
+		dst = &phash->expand[i];
+		src = ( dst - ( i / 8 ) );
+		*dst = ( ( ( src[-1] << 8 ) | src[0] ) >> ( i % 8 ) );
+	}
+	DBGC ( ctx, "MSCHAPv2 expanded password hash:\n" );
+	DBGC_HDA ( ctx, 0, phash->expand, sizeof ( phash->expand ) );
+}
+
+/**
+ * Calculate MS-CHAPv2 challenge response
+ *
+ * @v ctx		Context block
+ * @v chash		Challenge hash
+ * @v phash		Password hash (after expansion)
+ * @v nt		NT response to fill in
+ *
+ * This is the ChallengeResponse() function as documented in RFC 2759
+ * section 8.5.
+ */
+static void
+mschapv2_challenge_response ( union mschapv2_context *ctx,
+			      const union mschapv2_challenge_hash *chash,
+			      const union mschapv2_password_hash *phash,
+			      struct mschapv2_nt_response *nt ) {
+	struct cipher_algorithm *des = &des_algorithm;
+	unsigned int i;
+	int rc;
+
+	/* Construct response.  The design of the algorithm here is
+	 * interesting, suggesting that an intern at Microsoft had
+	 * heard the phrase "Triple DES" and hazarded a blind guess at
+	 * what it might mean.
+	 */
+	for ( i = 0 ; i < ( sizeof ( phash->des ) /
+			    sizeof ( phash->des[0] ) ) ; i++ ) {
+		rc = cipher_setkey ( des, ctx->des, phash->des[i],
+				     sizeof ( phash->des[i] ) );
+		assert ( rc == 0 ); /* no failure mode exists */
+		cipher_encrypt ( des, ctx->des, chash->des, nt->block[i],
+				 sizeof ( chash->des ) );
+	}
+	DBGC ( ctx, "MSCHAPv2 NT response:\n" );
+	DBGC_HDA ( ctx, 0, nt, sizeof ( *nt ) );
+}
+
+/**
+ * Calculate MS-CHAPv2 challenge response
+ *
+ * @v username		User name (or NULL to use empty string)
+ * @v password		Password (or NULL to use empty string)
+ * @v challenge		Authenticator challenge
+ * @v peer		Peer challenge
+ * @v response		Challenge response to fill in
+ *
+ * This is essentially the GenerateNTResponse() function as documented
+ * in RFC 2759 section 8.1.
+ */
+void mschapv2_response ( const char *username, const char *password,
+			 const struct mschapv2_challenge *challenge,
+			 const struct mschapv2_challenge *peer,
+			 struct mschapv2_response *response ) {
+	union mschapv2_context ctx;
+	union mschapv2_challenge_hash chash;
+	union mschapv2_password_hash phash;
+
+	/* Zero reserved fields */
+	memset ( response, 0, sizeof ( *response ) );
+
+	/* Copy peer challenge to response */
+	memcpy ( &response->peer, peer, sizeof ( response->peer ) );
+
+	/* Construct challenge hash */
+	mschapv2_challenge_hash ( &ctx, challenge, peer, username, &chash );
+
+	/* Construct expanded password hash */
+	mschapv2_password_hash ( &ctx, password, &phash );
+	mschapv2_expand_hash ( &ctx, &phash );
+
+	/* Construct NT response */
+	mschapv2_challenge_response ( &ctx, &chash, &phash, &response->nt );
+	DBGC ( &ctx, "MSCHAPv2 challenge response:\n" );
+	DBGC_HDA ( &ctx, 0, response, sizeof ( *response ) );
+}
+
+/**
+ * Calculate MS-CHAPv2 authenticator response
+ *
+ * @v username		User name (or NULL to use empty string)
+ * @v password		Password (or NULL to use empty string)
+ * @v challenge		Authenticator challenge
+ * @v response		Challenge response
+ * @v auth		Authenticator response to fill in
+ *
+ * This is essentially the GenerateAuthenticatorResponse() function as
+ * documented in RFC 2759 section 8.7.
+ */
+void mschapv2_auth ( const char *username, const char *password,
+		     const struct mschapv2_challenge *challenge,
+		     const struct mschapv2_response *response,
+		     struct mschapv2_auth *auth ) {
+	struct digest_algorithm *sha1 = &sha1_algorithm;
+	union mschapv2_context ctx;
+	union mschapv2_challenge_hash chash;
+	union mschapv2_password_hash phash;
+	char tmp[3];
+	char *wtf;
+	unsigned int i;
+
+	/* Construct hash of password hash */
+	mschapv2_password_hash ( &ctx, password, &phash );
+	mschapv2_hash_hash ( &ctx, &phash );
+
+	/* Construct unnamed intermediate hash */
+	digest_init ( sha1, ctx.sha1 );
+	digest_update ( sha1, ctx.sha1, phash.md4, sizeof ( phash.md4 ) );
+	digest_update ( sha1, ctx.sha1, &response->nt,
+			sizeof ( response->nt ) );
+	digest_update ( sha1, ctx.sha1, mschapv2_magic1,
+			sizeof ( mschapv2_magic1 ) );
+	digest_final ( sha1, ctx.sha1, phash.sha1 );
+	DBGC ( &ctx, "MSCHAPv2 NT response:\n" );
+	DBGC_HDA ( &ctx, 0, &response->nt, sizeof ( response->nt ) );
+	DBGC ( &ctx, "MSCHAPv2 unnamed intermediate hash:\n" );
+	DBGC_HDA ( &ctx, 0, phash.sha1, sizeof ( phash.sha1 ) );
+
+	/* Construct challenge hash */
+	mschapv2_challenge_hash ( &ctx, challenge, &response->peer,
+				  username, &chash );
+
+	/* Construct authenticator response hash */
+	digest_init ( sha1, ctx.sha1 );
+	digest_update ( sha1, ctx.sha1, phash.sha1, sizeof ( phash.sha1 ) );
+	digest_update ( sha1, ctx.sha1, chash.des, sizeof ( chash.des ) );
+	digest_update ( sha1, ctx.sha1, mschapv2_magic2,
+			sizeof ( mschapv2_magic2 ) );
+	digest_final ( sha1, ctx.sha1, phash.sha1 );
+	DBGC ( &ctx, "MSCHAPv2 authenticator response hash:\n" );
+	DBGC_HDA ( &ctx, 0, phash.sha1, sizeof ( phash.sha1 ) );
+
+	/* Encode authenticator response hash */
+	wtf = auth->wtf;
+	*(wtf++) = 'S';
+	*(wtf++) = '=';
+	DBGC ( &ctx, "MSCHAPv2 authenticator response: S=" );
+	for ( i = 0 ; i < sizeof ( phash.sha1 ) ; i++ ) {
+		snprintf ( tmp, sizeof ( tmp ), "%02X", phash.sha1[i] );
+		*(wtf++) = tmp[0];
+		*(wtf++) = tmp[1];
+		DBGC ( &ctx, "%s", tmp );
+	}
+	DBGC ( &ctx, "\n" );
+}
diff --git a/src/crypto/sha1.c b/src/crypto/sha1.c
index 94fce002..8eecc75b 100644
--- a/src/crypto/sha1.c
+++ b/src/crypto/sha1.c
@@ -145,12 +145,12 @@ static void sha1_digest ( struct sha1_context *context ) {
 
 	/* Sanity checks */
 	assert ( ( context->len % sizeof ( context->ddd.dd.data ) ) == 0 );
-	linker_assert ( &u.ddd.dd.digest.h[0] == a, sha1_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[1] == b, sha1_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[2] == c, sha1_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[3] == d, sha1_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[4] == e, sha1_bad_layout );
-	linker_assert ( &u.ddd.dd.data.dword[0] == w, sha1_bad_layout );
+	build_assert ( &u.ddd.dd.digest.h[0] == a );
+	build_assert ( &u.ddd.dd.digest.h[1] == b );
+	build_assert ( &u.ddd.dd.digest.h[2] == c );
+	build_assert ( &u.ddd.dd.digest.h[3] == d );
+	build_assert ( &u.ddd.dd.digest.h[4] == e );
+	build_assert ( &u.ddd.dd.data.dword[0] == w );
 
 	DBGC ( context, "SHA1 digesting:\n" );
 	DBGC_HDA ( context, 0, &context->ddd.dd.digest,
diff --git a/src/crypto/sha256.c b/src/crypto/sha256.c
index 6bd72771..c30300eb 100644
--- a/src/crypto/sha256.c
+++ b/src/crypto/sha256.c
@@ -140,15 +140,15 @@ static void sha256_digest ( struct sha256_context *context ) {
 
 	/* Sanity checks */
 	assert ( ( context->len % sizeof ( context->ddd.dd.data ) ) == 0 );
-	linker_assert ( &u.ddd.dd.digest.h[0] == a, sha256_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[1] == b, sha256_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[2] == c, sha256_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[3] == d, sha256_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[4] == e, sha256_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[5] == f, sha256_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[6] == g, sha256_bad_layout );
-	linker_assert ( &u.ddd.dd.digest.h[7] == h, sha256_bad_layout );
-	linker_assert ( &u.ddd.dd.data.dword[0] == w, sha256_bad_layout );
+	build_assert ( &u.ddd.dd.digest.h[0] == a );
+	build_assert ( &u.ddd.dd.digest.h[1] == b );
+	build_assert ( &u.ddd.dd.digest.h[2] == c );
+	build_assert ( &u.ddd.dd.digest.h[3] == d );
+	build_assert ( &u.ddd.dd.digest.h[4] == e );
+	build_assert ( &u.ddd.dd.digest.h[5] == f );
+	build_assert ( &u.ddd.dd.digest.h[6] == g );
+	build_assert ( &u.ddd.dd.digest.h[7] == h );
+	build_assert ( &u.ddd.dd.data.dword[0] == w );
 
 	DBGC ( context, "SHA256 digesting:\n" );
 	DBGC_HDA ( context, 0, &context->ddd.dd.digest,
diff --git a/src/crypto/sha512.c b/src/crypto/sha512.c
index e8489501..d7d44b28 100644
--- a/src/crypto/sha512.c
+++ b/src/crypto/sha512.c
@@ -156,15 +156,15 @@ static void sha512_digest ( struct sha512_context *context ) {
 
 	/* Sanity checks */
 	assert ( ( context->len % sizeof ( context->ddq.dd.data ) ) == 0 );
-	linker_assert ( &u.ddq.dd.digest.h[0] == a, sha512_bad_layout );
-	linker_assert ( &u.ddq.dd.digest.h[1] == b, sha512_bad_layout );
-	linker_assert ( &u.ddq.dd.digest.h[2] == c, sha512_bad_layout );
-	linker_assert ( &u.ddq.dd.digest.h[3] == d, sha512_bad_layout );
-	linker_assert ( &u.ddq.dd.digest.h[4] == e, sha512_bad_layout );
-	linker_assert ( &u.ddq.dd.digest.h[5] == f, sha512_bad_layout );
-	linker_assert ( &u.ddq.dd.digest.h[6] == g, sha512_bad_layout );
-	linker_assert ( &u.ddq.dd.digest.h[7] == h, sha512_bad_layout );
-	linker_assert ( &u.ddq.dd.data.qword[0] == w, sha512_bad_layout );
+	build_assert ( &u.ddq.dd.digest.h[0] == a );
+	build_assert ( &u.ddq.dd.digest.h[1] == b );
+	build_assert ( &u.ddq.dd.digest.h[2] == c );
+	build_assert ( &u.ddq.dd.digest.h[3] == d );
+	build_assert ( &u.ddq.dd.digest.h[4] == e );
+	build_assert ( &u.ddq.dd.digest.h[5] == f );
+	build_assert ( &u.ddq.dd.digest.h[6] == g );
+	build_assert ( &u.ddq.dd.digest.h[7] == h );
+	build_assert ( &u.ddq.dd.data.qword[0] == w );
 
 	DBGC ( context, "SHA512 digesting:\n" );
 	DBGC_HDA ( context, 0, &context->ddq.dd.digest,
diff --git a/src/crypto/x25519.c b/src/crypto/x25519.c
new file mode 100644
index 00000000..d58f7168
--- /dev/null
+++ b/src/crypto/x25519.c
@@ -0,0 +1,844 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * X25519 key exchange
+ *
+ * This implementation is inspired by and partially based upon the
+ * paper "Implementing Curve25519/X25519: A Tutorial on Elliptic Curve
+ * Cryptography" by Martin Kleppmann, available for download from
+ * https://www.cl.cam.ac.uk/teaching/2122/Crypto/curve25519.pdf
+ *
+ * The underlying modular addition, subtraction, and multiplication
+ * operations are completely redesigned for substantially improved
+ * efficiency compared to the TweetNaCl implementation studied in that
+ * paper.
+ *
+ *				       TweetNaCl	    iPXE
+ *				       ---------	    ----
+ *
+ * Storage size of each big integer	     128	      40
+ * (in bytes)
+ *
+ * Stack usage for key exchange		    1144	     360
+ * (in bytes, large objects only)
+ *
+ * Cost of big integer addition		      16	       5
+ * (in number of 64-bit additions)
+ *
+ * Cost of big integer multiplication	     273	      31
+ * (in number of 64-bit multiplications)
+ *
+ * The implementation is constant-time (provided that the underlying
+ * big integer operations are also constant-time).
+ */
+
+#include <stdint.h>
+#include <string.h>
+#include <assert.h>
+#include <errno.h>
+#include <ipxe/init.h>
+#include <ipxe/crypto.h>
+#include <ipxe/x25519.h>
+
+/** X25519 reduction constant
+ *
+ * The X25519 field prime is p=2^255-19.  This gives us:
+ *
+ *               p = 2^255 - 19
+ *           2^255 = p + 19
+ *           2^255 = 19          (mod p)
+ *       k * 2^255 = k * 19      (mod p)
+ *
+ * We can therefore reduce a value modulo p by taking the high-order
+ * bits of the value from bit 255 and above, multiplying by 19, and
+ * adding this to the low-order 255 bits of the value.
+ *
+ * This would be cumbersome to do in practice since it would require
+ * partitioning the value at a 255-bit boundary (and hence would
+ * require some shifting and masking operations).  However, we can
+ * note that:
+ *
+ *       k * 2^255 = k * 19      (mod p)
+ *   k * 2 * 2^255 = k * 2 * 19  (mod p)
+ *       k * 2^256 = k * 38      (mod p)
+ *
+ * We can therefore simplify the reduction to taking the high order
+ * bits of the value from bit 256 and above, multiplying by 38, and
+ * adding this to the low-order 256 bits of the value.
+ *
+ * Since 256 will inevitably be a multiple of the big integer element
+ * size (typically 32 or 64 bits), this avoids the need to perform any
+ * shifting or masking operations.
+ */
+#define X25519_REDUCE_256 38
+
+/** X25519 multiplication step 1 result
+ *
+ * Step 1 of X25519 multiplication is to compute the product of two
+ * X25519 unsigned 258-bit integers.
+ *
+ * Both multiplication inputs are limited to 258 bits, and so the
+ * product will have at most 516 bits.
+ */
+union x25519_multiply_step1 {
+	/** Raw product
+	 *
+	 * Big integer multiplication produces a result with a number
+	 * of elements equal to the sum of the number of elements in
+	 * each input.
+	 */
+	bigint_t ( X25519_SIZE + X25519_SIZE ) product;
+	/** Partition into low-order and high-order bits
+	 *
+	 * Reduction modulo p requires separating the low-order 256
+	 * bits from the remaining high-order bits.
+	 *
+	 * Since the value will never exceed 516 bits (see above),
+	 * there will be at most 260 high-order bits.
+	 */
+	struct {
+		/** Low-order 256 bits */
+		bigint_t ( bigint_required_size ( ( 256 /* bits */ + 7 ) / 8 ) )
+			low_256bit;
+		/** High-order 260 bits */
+		bigint_t ( bigint_required_size ( ( 260 /* bits */ + 7 ) / 8 ) )
+			high_260bit;
+	} __attribute__ (( packed )) parts;
+};
+
+/** X25519 multiplication step 2 result
+ *
+ * Step 2 of X25519 multiplication is to multiply the high-order 260
+ * bits from step 1 with the 6-bit reduction constant 38, and to add
+ * this to the low-order 256 bits from step 1.
+ *
+ * The multiplication inputs are limited to 260 and 6 bits
+ * respectively, and so the product will have at most 266 bits.  After
+ * adding the low-order 256 bits from step 1, the result will have at
+ * most 267 bits.
+ */
+union x25519_multiply_step2 {
+	/** Raw product
+	 *
+	 * Big integer multiplication produces a result with a number
+	 * of elements equal to the sum of the number of elements in
+	 * each input.
+	 */
+	bigint_t ( bigint_required_size ( ( 260 /* bits */ + 7 ) / 8 ) +
+		   bigint_required_size ( ( 6 /* bits */ + 7 ) / 8 ) ) product;
+	/** Big integer value
+	 *
+	 * The value will never exceed 267 bits (see above), and so
+	 * may be consumed as a normal X25519 big integer.
+	 */
+	x25519_t value;
+	/** Partition into low-order and high-order bits
+	 *
+	 * Reduction modulo p requires separating the low-order 256
+	 * bits from the remaining high-order bits.
+	 *
+	 * Since the value will never exceed 267 bits (see above),
+	 * there will be at most 11 high-order bits.
+	 */
+	struct {
+		/** Low-order 256 bits */
+		bigint_t ( bigint_required_size ( ( 256 /* bits */ + 7 ) / 8 ) )
+			low_256bit;
+		/** High-order 11 bits */
+		bigint_t ( bigint_required_size ( ( 11 /* bits */ + 7 ) / 8 ) )
+			high_11bit;
+	} __attribute__ (( packed )) parts;
+};
+
+/** X25519 multiplication step 3 result
+ *
+ * Step 3 of X25519 multiplication is to multiply the high-order 11
+ * bits from step 2 with the 6-bit reduction constant 38, and to add
+ * this to the low-order 256 bits from step 2.
+ *
+ * The multiplication inputs are limited to 11 and 6 bits
+ * respectively, and so the product will have at most 17 bits.  After
+ * adding the low-order 256 bits from step 2, the result will have at
+ * most 257 bits.
+ */
+union x25519_multiply_step3 {
+	/** Raw product
+	 *
+	 * Big integer multiplication produces a result with a number
+	 * of elements equal to the sum of the number of elements in
+	 * each input.
+	 */
+	bigint_t ( bigint_required_size ( ( 11 /* bits */ + 7 ) / 8 ) +
+		   bigint_required_size ( ( 6 /* bits */ + 7 ) / 8 ) ) product;
+	/** Big integer value
+	 *
+	 * The value will never exceed 267 bits (see above), and so
+	 * may be consumed as a normal X25519 big integer.
+	 */
+	x25519_t value;
+};
+
+/** X25519 multiplication temporary working space
+ *
+ * We overlap the buffers used by each step of the multiplication
+ * calculation to reduce the total stack space required:
+ *
+ * |--------------------------------------------------------|
+ * | <- pad -> | <------------ step 1 result -------------> |
+ * |           | <- low 256 bits -> | <-- high 260 bits --> |
+ * | <------- step 2 result ------> | <-- step 3 result --> |
+ * |--------------------------------------------------------|
+ */
+union x25519_multiply_workspace {
+	/** Step 1 result */
+	struct {
+		/** Padding to avoid collision between steps 1 and 2
+		 *
+		 * The step 2 multiplication consumes the high 260
+		 * bits of step 1, and so the step 2 multiplication
+		 * result must not overlap this portion of the step 1
+		 * result.
+		 */
+		uint8_t pad[ sizeof ( union x25519_multiply_step2 ) -
+			     offsetof ( union x25519_multiply_step1,
+					parts.high_260bit ) ];
+		/** Step 1 result */
+		union x25519_multiply_step1 step1;
+	} __attribute__ (( packed ));
+	/** Steps 2 and 3 results */
+	struct {
+		/** Step 2 result */
+		union x25519_multiply_step2 step2;
+		/** Step 3 result */
+		union x25519_multiply_step3 step3;
+	} __attribute__ (( packed ));
+};
+
+/** An X25519 elliptic curve point in projective coordinates
+ *
+ * A point (x,y) on the Montgomery curve used in X25519 is represented
+ * using projective coordinates (X/Z,Y/Z) so that intermediate
+ * calculations may be performed on both numerator and denominator
+ * separately, with the division step performed only once at the end
+ * of the calculation.
+ *
+ * The group operation calculation is performed using a Montgomery
+ * ladder as:
+ *
+ *   X[2i]   = ( X[i]^2 - Z[i]^2 )^2
+ *   X[2i+1] = ( X[i] * X[i+1] - Z[i] * Z[i+1] )^2
+ *   Z[2i]   = 4 * X[i] * Z[i] * ( X[i]^2 + A * X[i] * Z[i] + Z[i]^2 )
+ *   Z[2i+1] = X[0] * ( X[i] * Z[i+1] - X[i+1] * Z[i] ) ^ 2
+ *
+ * It is therefore not necessary to store (or use) the value of Y.
+ */
+struct x25519_projective {
+	/** X coordinate */
+	union x25519_quad257 X;
+	/** Z coordinate */
+	union x25519_quad257 Z;
+};
+
+/** An X25519 Montgomery ladder step */
+struct x25519_step {
+	/** X[n]/Z[n] */
+	struct x25519_projective x_n;
+	/** X[n+1]/Z[n+1] */
+	struct x25519_projective x_n1;
+};
+
+/** Constant p=2^255-19 (the finite field prime) */
+static const uint8_t x25519_p_raw[] = {
+	0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xed
+};
+
+/** Constant p=2^255-19 (the finite field prime) */
+static x25519_t x25519_p;
+
+/** Constant 2p=2^256-38 */
+static x25519_t x25519_2p;
+
+/** Constant 4p=2^257-76 */
+static x25519_t x25519_4p;
+
+/** Reduction constant (used during multiplication) */
+static const uint8_t x25519_reduce_256_raw[] = { X25519_REDUCE_256 };
+
+/** Reduction constant (used during multiplication) */
+static bigint_t ( bigint_required_size ( sizeof ( x25519_reduce_256_raw ) ) )
+	x25519_reduce_256;
+
+/** Constant 121665 (used in the Montgomery ladder) */
+static const uint8_t x25519_121665_raw[] = { 0x01, 0xdb, 0x41 };
+
+/** Constant 121665 (used in the Montgomery ladder) */
+static union x25519_oct258 x25519_121665;
+
+/** Constant g=9 (the group generator) */
+static struct x25519_value x25519_generator = {
+	.raw = { 9, }
+};
+
+/**
+ * Initialise constants
+ *
+ */
+static void x25519_init_constants ( void ) {
+
+	/* Construct constant p */
+	bigint_init ( &x25519_p, x25519_p_raw, sizeof ( x25519_p_raw ) );
+
+	/* Construct constant 2p */
+	bigint_copy ( &x25519_p, &x25519_2p );
+	bigint_add ( &x25519_p, &x25519_2p );
+
+	/* Construct constant 4p */
+	bigint_copy ( &x25519_2p, &x25519_4p );
+	bigint_add ( &x25519_2p, &x25519_4p );
+
+	/* Construct reduction constant */
+	bigint_init ( &x25519_reduce_256, x25519_reduce_256_raw,
+		      sizeof ( x25519_reduce_256_raw ) );
+
+	/* Construct constant 121665 */
+	bigint_init ( &x25519_121665.value, x25519_121665_raw,
+		      sizeof ( x25519_121665_raw ) );
+}
+
+/** Initialisation function */
+struct init_fn x25519_init_fn __init_fn ( INIT_NORMAL ) = {
+	.initialise = x25519_init_constants,
+};
+
+/**
+ * Add big integers modulo field prime
+ *
+ * @v augend		Big integer to add
+ * @v addend		Big integer to add
+ * @v result		Big integer to hold result (may overlap augend)
+ */
+static inline __attribute__ (( always_inline )) void
+x25519_add ( const union x25519_quad257 *augend,
+	     const union x25519_quad257 *addend,
+	     union x25519_oct258 *result ) {
+	int copy;
+
+	/* Copy augend if necessary */
+	copy = ( result != &augend->oct258 );
+	build_assert ( __builtin_constant_p ( copy ) );
+	if ( copy ) {
+		build_assert ( result != &addend->oct258 );
+		bigint_copy ( &augend->oct258.value, &result->value );
+	}
+
+	/* Perform addition
+	 *
+	 * Both inputs are in the range [0,4p-1] and the resulting
+	 * sum is therefore in the range [0,8p-2].
+	 *
+	 * This range lies within the range [0,8p-1] and the result is
+	 * therefore a valid X25519 unsigned 258-bit integer, as
+	 * required.
+	 */
+	bigint_add ( &addend->value, &result->value );
+}
+
+/**
+ * Subtract big integers modulo field prime
+ *
+ * @v minuend		Big integer from which to subtract
+ * @v subtrahend	Big integer to subtract
+ * @v result		Big integer to hold result (may overlap minuend)
+ */
+static inline __attribute__ (( always_inline )) void
+x25519_subtract ( const union x25519_quad257 *minuend,
+		  const union x25519_quad257 *subtrahend,
+		  union x25519_oct258 *result ) {
+	int copy;
+
+	/* Copy minuend if necessary */
+	copy = ( result != &minuend->oct258 );
+	build_assert ( __builtin_constant_p ( copy ) );
+	if ( copy ) {
+		build_assert ( result != &subtrahend->oct258 );
+		bigint_copy ( &minuend->oct258.value, &result->value );
+	}
+
+	/* Perform subtraction
+	 *
+	 * Both inputs are in the range [0,4p-1] and the resulting
+	 * difference is therefore in the range [1-4p,4p-1].
+	 *
+	 * This range lies partially outside the range [0,8p-1] and
+	 * the result is therefore not yet a valid X25519 unsigned
+	 * 258-bit integer.
+	 */
+	bigint_subtract ( &subtrahend->value, &result->value );
+
+	/* Add constant multiple of field prime p
+	 *
+	 * Add the constant 4p to the result.  This brings the result
+	 * within the range [1,8p-1] (without changing the value
+	 * modulo p).
+	 *
+	 * This range lies within the range [0,8p-1] and the result is
+	 * therefore now a valid X25519 unsigned 258-bit integer, as
+	 * required.
+	 */
+	bigint_add ( &x25519_4p, &result->value );
+}
+
+/**
+ * Multiply big integers modulo field prime
+ *
+ * @v multiplicand	Big integer to be multiplied
+ * @v multiplier	Big integer to be multiplied
+ * @v result		Big integer to hold result (may overlap either input)
+ */
+void x25519_multiply ( const union x25519_oct258 *multiplicand,
+		       const union x25519_oct258 *multiplier,
+		       union x25519_quad257 *result ) {
+	union x25519_multiply_workspace tmp;
+	union x25519_multiply_step1 *step1 = &tmp.step1;
+	union x25519_multiply_step2 *step2 = &tmp.step2;
+	union x25519_multiply_step3 *step3 = &tmp.step3;
+
+	/* Step 1: perform raw multiplication
+	 *
+	 *   step1 = multiplicand * multiplier
+	 *
+	 * Both inputs are 258-bit numbers and the step 1 result is
+	 * therefore 258+258=516 bits.
+	 */
+	static_assert ( sizeof ( step1->product ) >= sizeof ( step1->parts ) );
+	bigint_multiply ( &multiplicand->value, &multiplier->value,
+			  &step1->product );
+
+	/* Step 2: reduce high-order 516-256=260 bits of step 1 result
+	 *
+	 * Use the identity 2^256=38 (mod p) to reduce the high-order
+	 * bits of the step 1 result.  We split the 516-bit result
+	 * from step 1 into its low-order 256 bits and high-order 260
+	 * bits:
+	 *
+	 *   step1 = step1(low 256 bits) + step1(high 260 bits) * 2^256
+	 *
+	 * and then perform the calculation:
+	 *
+	 *   step2 = step1                                              (mod p)
+	 *         = step1(low 256 bits) + step1(high 260 bits) * 2^256 (mod p)
+	 *         = step1(low 256 bits) + step1(high 260 bits) * 38    (mod p)
+	 *
+	 * There are 6 bits in the constant value 38.  The step 2
+	 * multiplication product will therefore have 260+6=266 bits,
+	 * and the step 2 result (after the addition) will therefore
+	 * have 267 bits.
+	 */
+	static_assert ( sizeof ( step2->product ) >= sizeof ( step2->value ) );
+	static_assert ( sizeof ( step2->product ) >= sizeof ( step2->parts ) );
+	bigint_grow ( &step1->parts.low_256bit, &result->value );
+	bigint_multiply ( &step1->parts.high_260bit, &x25519_reduce_256,
+			  &step2->product );
+	bigint_add ( &result->value, &step2->value );
+
+	/* Step 3: reduce high-order 267-256=11 bits of step 2 result
+	 *
+	 * Use the identity 2^256=38 (mod p) again to reduce the
+	 * high-order bits of the step 2 result.  As before, we split
+	 * the 267-bit result from step 2 into its low-order 256 bits
+	 * and high-order 11 bits:
+	 *
+	 *   step2 = step2(low 256 bits) + step2(high 11 bits) * 2^256
+	 *
+	 * and then perform the calculation:
+	 *
+	 *   step3 = step2                                             (mod p)
+	 *         = step2(low 256 bits) + step2(high 11 bits) * 2^256 (mod p)
+	 *         = step2(low 256 bits) + step2(high 11 bits) * 38    (mod p)
+	 *
+	 * There are 6 bits in the constant value 38.  The step 3
+	 * multiplication product will therefore have 11+6=19 bits,
+	 * and the step 3 result (after the addition) will therefore
+	 * have 257 bits.
+	 *
+	 * A loose upper bound for the step 3 result (after the
+	 * addition) is given by:
+	 *
+	 *   step3 < ( 2^256 - 1 ) + ( 2^19 - 1 )
+	 *         < ( 2^257 - 2^256 - 1 ) + ( 2^19 - 1 )
+	 *         < ( 2^257 - 76 ) - 2^256 + 2^19 + 74
+	 *         < 4 * ( 2^255 - 19 ) - 2^256 + 2^19 + 74
+	 *         < 4p - 2^256 + 2^19 + 74
+	 *
+	 * and so the step 3 result is strictly less than 4p, and
+	 * therefore lies within the range [0,4p-1].
+	 */
+	memset ( &step3->value, 0, sizeof ( step3->value ) );
+	bigint_grow ( &step2->parts.low_256bit, &result->value );
+	bigint_multiply ( &step2->parts.high_11bit, &x25519_reduce_256,
+			  &step3->product );
+	bigint_add ( &step3->value, &result->value );
+
+	/* Step 1 calculates the product of the input operands, and
+	 * each subsequent step reduces the number of bits in the
+	 * result while preserving this value (modulo p).  The final
+	 * result is therefore equal to the product of the input
+	 * operands (modulo p), as required.
+	 *
+	 * The step 3 result lies within the range [0,4p-1] and the
+	 * final result is therefore a valid X25519 unsigned 257-bit
+	 * integer, as required.
+	 */
+}
+
+/**
+ * Compute multiplicative inverse
+ *
+ * @v invertend		Big integer to be inverted
+ * @v result		Big integer to hold result (may not overlap input)
+ */
+void x25519_invert ( const union x25519_oct258 *invertend,
+		     union x25519_quad257 *result ) {
+	int i;
+
+	/* Sanity check */
+	assert ( invertend != &result->oct258 );
+
+	/* Calculate inverse as x^(-1)=x^(p-2) where p is the field prime
+	 *
+	 * The field prime is p=2^255-19 and so:
+	 *
+	 *   p - 2 = 2^255 - 21
+	 *         = (2^255 - 1) - 2^4 - 2^2
+	 *
+	 * i.e. p-2 is a 254-bit number in which all bits are set
+	 * apart from bit 2 and bit 4.
+	 *
+	 * We use the square-and-multiply method to compute x^(p-2).
+	 */
+	bigint_copy ( &invertend->value, &result->value );
+	for ( i = 253 ; i >= 0 ; i-- ) {
+
+		/* Square running total */
+		x25519_multiply ( &result->oct258, &result->oct258, result );
+
+		/* For each set bit in the exponent, multiply by invertend */
+		if ( ( i != 2 ) && ( i != 4 ) ) {
+			x25519_multiply ( invertend, &result->oct258, result );
+		}
+	}
+}
+
+/**
+ * Reduce big integer via conditional subtraction
+ *
+ * @v subtrahend	Big integer to subtract
+ * @v value		Big integer to be subtracted from, if possible
+ */
+static void x25519_reduce_by ( const x25519_t *subtrahend, x25519_t *value ) {
+	unsigned int max_bit = ( ( 8 * sizeof ( *value ) ) - 1 );
+	x25519_t tmp;
+
+	/* Conditionally subtract subtrahend
+	 *
+	 * Subtract the subtrahend, discarding the result (in constant
+	 * time) if the subtraction underflows.
+	 */
+	bigint_copy ( value, &tmp );
+	bigint_subtract ( subtrahend, value );
+	bigint_swap ( value, &tmp, bigint_bit_is_set ( value, max_bit ) );
+}
+
+/**
+ * Reduce big integer to canonical range
+ *
+ * @v value		Big integer to be reduced
+ */
+void x25519_reduce ( union x25519_quad257 *value ) {
+
+	/* Conditionally subtract 2p
+	 *
+	 * Subtract twice the field prime, discarding the result (in
+	 * constant time) if the subtraction underflows.
+	 *
+	 * The input value is in the range [0,4p-1].  After this
+	 * conditional subtraction, the value is in the range
+	 * [0,2p-1].
+	 */
+	x25519_reduce_by ( &x25519_2p, &value->value );
+
+	/* Conditionally subtract p
+	 *
+	 * Subtract the field prime, discarding the result (in
+	 * constant time) if the subtraction underflows.
+	 *
+	 * The value is already in the range [0,2p-1].  After this
+	 * conditional subtraction, the value is in the range [0,p-1]
+	 * and is therefore the canonical representation.
+	 */
+	x25519_reduce_by ( &x25519_p, &value->value );
+}
+
+/**
+ * Compute next step of the Montgomery ladder
+ *
+ * @v base		Base point
+ * @v bit		Bit value
+ * @v step		Ladder step
+ */
+static void x25519_step ( const union x25519_quad257 *base, int bit,
+			  struct x25519_step *step ) {
+	union x25519_quad257 *a = &step->x_n.X;
+	union x25519_quad257 *b = &step->x_n1.X;
+	union x25519_quad257 *c = &step->x_n.Z;
+	union x25519_quad257 *d = &step->x_n1.Z;
+	union x25519_oct258 e;
+	union x25519_quad257 f;
+	union x25519_oct258 *v1_e;
+	union x25519_oct258 *v2_a;
+	union x25519_oct258 *v3_c;
+	union x25519_oct258 *v4_b;
+	union x25519_quad257 *v5_d;
+	union x25519_quad257 *v6_f;
+	union x25519_quad257 *v7_a;
+	union x25519_quad257 *v8_c;
+	union x25519_oct258 *v9_e;
+	union x25519_oct258 *v10_a;
+	union x25519_quad257 *v11_b;
+	union x25519_oct258 *v12_c;
+	union x25519_quad257 *v13_a;
+	union x25519_oct258 *v14_a;
+	union x25519_quad257 *v15_c;
+	union x25519_quad257 *v16_a;
+	union x25519_quad257 *v17_d;
+	union x25519_quad257 *v18_b;
+
+	/* See the referenced paper "Implementing Curve25519/X25519: A
+	 * Tutorial on Elliptic Curve Cryptography" for the reasoning
+	 * behind this calculation.
+	 */
+
+	/* Reuse storage locations for intermediate results where possible */
+	v1_e = &e;
+	v2_a = container_of ( &a->value, union x25519_oct258, value );
+	v3_c = container_of ( &c->value, union x25519_oct258, value );
+	v4_b = container_of ( &b->value, union x25519_oct258, value );
+	v5_d = d;
+	v6_f = &f;
+	v7_a = a;
+	v8_c = c;
+	v9_e = &e;
+	v10_a = container_of ( &a->value, union x25519_oct258, value );
+	v11_b = b;
+	v12_c = container_of ( &c->value, union x25519_oct258, value );
+	v13_a = a;
+	v14_a = container_of ( &a->value, union x25519_oct258, value );
+	v15_c = c;
+	v16_a = a;
+	v17_d = d;
+	v18_b = b;
+
+	/* Select inputs */
+	bigint_swap ( &a->value, &b->value, bit );
+	bigint_swap ( &c->value, &d->value, bit );
+
+	/* v1 = a + c */
+	x25519_add ( a, c, v1_e );
+
+	/* v2 = a - c */
+	x25519_subtract ( a, c, v2_a );
+
+	/* v3 = b + d */
+	x25519_add ( b, d, v3_c );
+
+	/* v4 = b - d */
+	x25519_subtract ( b, d, v4_b );
+
+	/* v5 = v1^2 = (a + c)^2 = a^2 + 2ac + c^2 */
+	x25519_multiply ( v1_e, v1_e, v5_d );
+
+	/* v6 = v2^2 = (a - c)^2 = a^2 - 2ac + c^2 */
+	x25519_multiply ( v2_a, v2_a, v6_f );
+
+	/* v7 = v3 * v2 = (b + d) * (a - c) = ab - bc + ad - cd */
+	x25519_multiply ( v3_c, v2_a, v7_a );
+
+	/* v8 = v4 * v1 = (b - d) * (a + c) = ab + bc - ad - cd */
+	x25519_multiply ( v4_b, v1_e, v8_c );
+
+	/* v9 = v7 + v8 = 2 * (ab - cd) */
+	x25519_add ( v7_a, v8_c, v9_e );
+
+	/* v10 = v7 - v8 = 2 * (ad - bc) */
+	x25519_subtract ( v7_a, v8_c, v10_a );
+
+	/* v11 = v10^2 = 4 * (ad - bc)^2 */
+	x25519_multiply ( v10_a, v10_a, v11_b );
+
+	/* v12 = v5 - v6 = (a + c)^2 - (a - c)^2 = 4ac */
+	x25519_subtract ( v5_d, v6_f, v12_c );
+
+	/* v13 = v12 * 121665 = 486660ac = (A-2) * ac */
+	x25519_multiply ( v12_c, &x25519_121665, v13_a );
+
+	/* v14 = v13 + v5 = (A-2) * ac + a^2 + 2ac + c^2 = a^2 + A * ac + c^2 */
+	x25519_add ( v13_a, v5_d, v14_a );
+
+	/* v15 = v12 * v14 = 4ac * (a^2 + A * ac + c^2) */
+	x25519_multiply ( v12_c, v14_a, v15_c );
+
+	/* v16 = v5 * v6 = (a + c)^2 * (a - c)^2 = (a^2 - c^2)^2 */
+	x25519_multiply ( &v5_d->oct258, &v6_f->oct258, v16_a );
+
+	/* v17 = v11 * base = 4 * base * (ad - bc)^2 */
+	x25519_multiply ( &v11_b->oct258, &base->oct258, v17_d );
+
+	/* v18 = v9^2 = 4 * (ab - cd)^2 */
+	x25519_multiply ( v9_e, v9_e, v18_b );
+
+	/* Select outputs */
+	bigint_swap ( &a->value, &b->value, bit );
+	bigint_swap ( &c->value, &d->value, bit );
+}
+
+/**
+ * Multiply X25519 elliptic curve point
+ *
+ * @v base		Base point
+ * @v scalar		Scalar multiple
+ * @v result		Point to hold result (may overlap base point)
+ */
+static void x25519_ladder ( const union x25519_quad257 *base,
+			    struct x25519_value *scalar,
+			    union x25519_quad257 *result ) {
+	static const uint8_t zero[] = { 0 };
+	static const uint8_t one[] = { 1 };
+	struct x25519_step step;
+	union x25519_quad257 *tmp;
+	int bit;
+	int i;
+
+	/* Initialise ladder */
+	bigint_init ( &step.x_n.X.value, one, sizeof ( one ) );
+	bigint_init ( &step.x_n.Z.value, zero, sizeof ( zero ) );
+	bigint_copy ( &base->value, &step.x_n1.X.value );
+	bigint_init ( &step.x_n1.Z.value, one, sizeof ( one ) );
+
+	/* Use ladder */
+	for ( i = 254 ; i >= 0 ; i-- ) {
+		bit = ( ( scalar->raw[ i / 8 ] >> ( i % 8 ) ) & 1 );
+		x25519_step ( base, bit, &step );
+	}
+
+	/* Convert back to affine coordinate */
+	tmp = &step.x_n1.X;
+	x25519_invert ( &step.x_n.Z.oct258, tmp );
+	x25519_multiply ( &step.x_n.X.oct258, &tmp->oct258, result );
+	x25519_reduce ( result );
+}
+
+/**
+ * Reverse X25519 value endianness
+ *
+ * @v value		Value to reverse
+ */
+static void x25519_reverse ( struct x25519_value *value ) {
+	uint8_t *low = value->raw;
+	uint8_t *high = &value->raw[ sizeof ( value->raw ) - 1 ];
+	uint8_t tmp;
+
+	/* Reverse bytes */
+	do {
+		tmp = *low;
+		*low = *high;
+		*high = tmp;
+	} while ( ++low < --high );
+}
+
+/**
+ * Calculate X25519 key
+ *
+ * @v base		Base point
+ * @v scalar		Scalar multiple
+ * @v result		Point to hold result (may overlap base point)
+ * @ret rc		Return status code
+ */
+int x25519_key ( const struct x25519_value *base,
+		 const struct x25519_value *scalar,
+		 struct x25519_value *result ) {
+	struct x25519_value *tmp = result;
+	union x25519_quad257 point;
+
+	/* Reverse base point and clear high bit as required by RFC7748 */
+	memcpy ( tmp, base, sizeof ( *tmp ) );
+	x25519_reverse ( tmp );
+	tmp->raw[0] &= 0x7f;
+	bigint_init ( &point.value, tmp->raw, sizeof ( tmp->raw ) );
+
+	/* Clamp scalar as required by RFC7748 */
+	memcpy ( tmp, scalar, sizeof ( *tmp ) );
+	tmp->raw[0] &= 0xf8;
+	tmp->raw[31] |= 0x40;
+
+	/* Multiply elliptic curve point */
+	x25519_ladder ( &point, tmp, &point );
+
+	/* Reverse result */
+	bigint_done ( &point.value, result->raw, sizeof ( result->raw ) );
+	x25519_reverse ( result );
+
+	/* Fail if result was all zeros (as required by RFC8422) */
+	return ( bigint_is_zero ( &point.value ) ? -EPERM : 0 );
+}
+
+/**
+ * Multiply scalar by curve point
+ *
+ * @v base		Base point (or NULL to use generator)
+ * @v scalar		Scalar multiple
+ * @v result		Result point to fill in
+ * @ret rc		Return status code
+ */
+static int x25519_curve_multiply ( const void *base, const void *scalar,
+				   void *result ) {
+
+	/* Use base point if applicable */
+	if ( ! base )
+		base = &x25519_generator;
+
+	return x25519_key ( base, scalar, result );
+}
+
+/** X25519 elliptic curve */
+struct elliptic_curve x25519_curve = {
+	.name = "x25519",
+	.keysize = sizeof ( struct x25519_value ),
+	.multiply = x25519_curve_multiply,
+};
diff --git a/src/crypto/x509.c b/src/crypto/x509.c
index 1f017eb0..92318093 100644
--- a/src/crypto/x509.c
+++ b/src/crypto/x509.c
@@ -1603,19 +1603,12 @@ int x509_check_name ( struct x509_certificate *cert, const char *name ) {
 static void x509_free_chain ( struct refcnt *refcnt ) {
 	struct x509_chain *chain =
 		container_of ( refcnt, struct x509_chain, refcnt );
-	struct x509_link *link;
-	struct x509_link *tmp;
 
 	DBGC2 ( chain, "X509 chain %p freed\n", chain );
 
-	/* Free each link in the chain */
-	list_for_each_entry_safe ( link, tmp, &chain->links, list ) {
-		x509_put ( link->cert );
-		list_del ( &link->list );
-		free ( link );
-	}
-
 	/* Free chain */
+	x509_truncate ( chain, NULL );
+	assert ( list_empty ( &chain->links ) );
 	free ( chain );
 }
 
@@ -1697,6 +1690,27 @@ int x509_append_raw ( struct x509_chain *chain, const void *data,
 }
 
 /**
+ * Truncate X.509 certificate chain
+ *
+ * @v chain		X.509 certificate chain
+ * @v link		Link after which to truncate chain, or NULL
+ */
+void x509_truncate ( struct x509_chain *chain, struct x509_link *link ) {
+	struct x509_link *tmp;
+
+	/* Truncate entire chain if no link is specified */
+	if ( ! link )
+		link = list_entry ( &chain->links, struct x509_link, list );
+
+	/* Free each link in the chain */
+	list_for_each_entry_safe_continue ( link, tmp, &chain->links, list ) {
+		x509_put ( link->cert );
+		list_del ( &link->list );
+		free ( link );
+	}
+}
+
+/**
  * Identify X.509 certificate by subject
  *
  * @v certs		X.509 certificate list
diff --git a/src/drivers/bus/ecam.c b/src/drivers/bus/ecam.c
index 1d57bd2a..5e3debdd 100644
--- a/src/drivers/bus/ecam.c
+++ b/src/drivers/bus/ecam.c
@@ -127,7 +127,7 @@ static int ecam_access ( struct pci_device *pci ) {
 
 	/* Reuse mapping if possible */
 	if ( ( pci->busdevfn - ecam.range.start ) < ecam.range.count )
-		return 0;
+		return ecam.rc;
 
 	/* Clear any existing mapping */
 	if ( ecam.regs ) {
@@ -145,12 +145,22 @@ static int ecam_access ( struct pci_device *pci ) {
 	if ( ecam.range.start > pci->busdevfn ) {
 		DBGC ( &ecam, "ECAM found no allocation for " PCI_FMT "\n",
 		       PCI_ARGS ( pci ) );
+		rc = -ENOENT;
 		goto err_find;
 	}
 
 	/* Map configuration space for this allocation */
 	base = le64_to_cpu ( ecam.alloc.base );
+	base += ( ecam.alloc.start * ECAM_SIZE * PCI_BUSDEVFN ( 0, 1, 0, 0 ) );
 	len = ( ecam.range.count * ECAM_SIZE );
+	if ( base != ( ( unsigned long ) base ) ) {
+		DBGC ( &ecam, "ECAM %04x:[%02x-%02x] could not map "
+		       "[%08llx,%08llx) outside CPU range\n",
+		       le16_to_cpu ( ecam.alloc.segment ), ecam.alloc.start,
+		       ecam.alloc.end, base, ( base + len ) );
+		rc = -ERANGE;
+		goto err_range;
+	}
 	ecam.regs = ioremap ( base, len );
 	if ( ! ecam.regs ) {
 		DBGC ( &ecam, "ECAM %04x:[%02x-%02x] could not map "
@@ -164,12 +174,14 @@ static int ecam_access ( struct pci_device *pci ) {
 	DBGC ( &ecam, "ECAM %04x:[%02x-%02x] mapped [%08llx,%08llx) -> %p\n",
 	       le16_to_cpu ( ecam.alloc.segment ), ecam.alloc.start,
 	       ecam.alloc.end, base, ( base + len ), ecam.regs );
+	ecam.rc = 0;
 	return 0;
 
 	iounmap ( ecam.regs );
  err_ioremap:
+ err_range:
  err_find:
-	ecam.range.count = 0;
+	ecam.rc = rc;
 	return rc;
 }
 
@@ -235,7 +247,7 @@ int ecam_write ( struct pci_device *pci, unsigned int location,
 	if ( ( rc = ecam_access ( pci ) ) != 0 )
 		return rc;
 
-	/* Read from address */
+	/* Write to address */
 	index = ( pci->busdevfn - ecam.range.start );
 	addr = ( ecam.regs + ( index * ECAM_SIZE ) + where );
 	switch ( len ) {
@@ -252,6 +264,15 @@ int ecam_write ( struct pci_device *pci, unsigned int location,
 		assert ( 0 );
 	}
 
+	/* Read from address, to guarantee completion of the write
+	 *
+	 * PCIe configuration space registers may not have read side
+	 * effects.  Reading back is therefore always safe to do, and
+	 * guarantees that the write has reached the device.
+	 */
+	mb();
+	ecam_read ( pci, location, &value );
+
 	return 0;
 }
 
diff --git a/src/drivers/infiniband/arbel.c b/src/drivers/infiniband/arbel.c
index 293c1b64..8be06d93 100644
--- a/src/drivers/infiniband/arbel.c
+++ b/src/drivers/infiniband/arbel.c
@@ -545,8 +545,8 @@ static int arbel_mad ( struct ib_device *ibdev, union ib_mad *mad ) {
 	union arbelprm_mad mad_ifc;
 	int rc;
 
-	linker_assert ( sizeof ( *mad ) == sizeof ( mad_ifc.mad ),
-			mad_size_mismatch );
+	/* Sanity check */
+	static_assert ( sizeof ( *mad ) == sizeof ( mad_ifc.mad ) );
 
 	/* Copy in request packet */
 	memcpy ( &mad_ifc.mad, mad, sizeof ( mad_ifc.mad ) );
@@ -3139,8 +3139,8 @@ static void arbel_remove ( struct pci_device *pci ) {
 }
 
 static struct pci_device_id arbel_nics[] = {
-	PCI_ROM ( 0x15b3, 0x6282, "mt25218", "MT25218 HCA driver", 0 ),
 	PCI_ROM ( 0x15b3, 0x6274, "mt25204", "MT25204 HCA driver", 0 ),
+	PCI_ROM ( 0x15b3, 0x6282, "mt25218", "MT25218 HCA driver", 0 ),
 };
 
 struct pci_driver arbel_driver __pci_driver = {
diff --git a/src/drivers/infiniband/hermon.c b/src/drivers/infiniband/hermon.c
index c09baf7a..e5c3544f 100644
--- a/src/drivers/infiniband/hermon.c
+++ b/src/drivers/infiniband/hermon.c
@@ -779,8 +779,8 @@ static int hermon_mad ( struct ib_device *ibdev, union ib_mad *mad ) {
 	union hermonprm_mad mad_ifc;
 	int rc;
 
-	linker_assert ( sizeof ( *mad ) == sizeof ( mad_ifc.mad ),
-			mad_size_mismatch );
+	/* Sanity check */
+	static_assert ( sizeof ( *mad ) == sizeof ( mad_ifc.mad ) );
 
 	/* Copy in request packet */
 	memcpy ( &mad_ifc.mad, mad, sizeof ( mad_ifc.mad ) );
@@ -4214,6 +4214,9 @@ static void hermon_bofm_remove ( struct pci_device *pci ) {
 }
 
 static struct pci_device_id hermon_nics[] = {
+	/* Mellanox ConnectX-3 VPI (ethernet + infiniband) */
+	PCI_ROM ( 0x15b3, 0x1003, "mt4099", "ConnectX-3 HCA driver", 0 ),
+	PCI_ROM ( 0x15b3, 0x1007, "mt4103", "ConnectX-3 Pro HCA driver", 0 ),
 	/* Mellanox ConnectX VPI (ethernet + infiniband) */
 	PCI_ROM ( 0x15b3, 0x6340, "mt25408", "MT25408 HCA driver", 0 ),
 	PCI_ROM ( 0x15b3, 0x634a, "mt25418", "MT25418 HCA driver", 0 ),
@@ -4226,17 +4229,13 @@ static struct pci_device_id hermon_nics[] = {
 	PCI_ROM ( 0x15b3, 0x6732, "mt26418", "MT26418 HCA driver", 0 ),
 	PCI_ROM ( 0x15b3, 0x673c, "mt26428", "MT26428 HCA driver", 0 ),
 	PCI_ROM ( 0x15b3, 0x6746, "mt26438", "MT26438 HCA driver", 0 ),
-	PCI_ROM ( 0x15b3, 0x6778, "mt26488", "MT26488 HCA driver", 0 ),
 
 	/* Mellanox ConnectX-2 EN (ethernet only) */
 	PCI_ROM ( 0x15b3, 0x6750, "mt26448", "MT26448 HCA driver", 0 ),
 	PCI_ROM ( 0x15b3, 0x675a, "mt26458", "MT26458 HCA driver", 0 ),
 	PCI_ROM ( 0x15b3, 0x6764, "mt26468", "MT26468 HCA driver", 0 ),
 	PCI_ROM ( 0x15b3, 0x676e, "mt26478", "MT26478 HCA driver", 0 ),
-
-	/* Mellanox ConnectX-3 VPI (ethernet + infiniband) */
-	PCI_ROM ( 0x15b3, 0x1003, "mt4099", "ConnectX-3 HCA driver", 0 ),
-	PCI_ROM ( 0x15b3, 0x1007, "mt4103", "ConnectX-3 Pro HCA driver", 0 ),
+	PCI_ROM ( 0x15b3, 0x6778, "mt26488", "MT26488 HCA driver", 0 ),
 };
 
 struct pci_driver hermon_driver __pci_driver = {
diff --git a/src/drivers/infiniband/linda.c b/src/drivers/infiniband/linda.c
index 8c591266..0c8a043a 100644
--- a/src/drivers/infiniband/linda.c
+++ b/src/drivers/infiniband/linda.c
@@ -721,7 +721,7 @@ static int linda_init_recv ( struct linda *linda ) {
 		eager_array_size_other = LINDA_EAGER_ARRAY_SIZE_17CTX_OTHER;
 		break;
 	default:
-		linker_assert ( 0, invalid_LINDA_NUM_CONTEXTS );
+		build_assert ( 0 );
 		return -EINVAL;
 	}
 
@@ -1108,7 +1108,7 @@ static int linda_post_recv ( struct ib_device *ibdev,
 	case 16384: bufsize = LINDA_EAGER_BUFFER_16K; break;
 	case 32768: bufsize = LINDA_EAGER_BUFFER_32K; break;
 	case 65536: bufsize = LINDA_EAGER_BUFFER_64K; break;
-	default:    linker_assert ( 0, invalid_rx_payload_size );
+	default:    build_assert ( 0 );
 		    bufsize = LINDA_EAGER_BUFFER_NONE;
 	}
 
diff --git a/src/drivers/infiniband/qib7322.c b/src/drivers/infiniband/qib7322.c
index da055b74..a011dafc 100644
--- a/src/drivers/infiniband/qib7322.c
+++ b/src/drivers/infiniband/qib7322.c
@@ -893,7 +893,7 @@ static int qib7322_init_recv ( struct qib7322 *qib7322 ) {
 		eager_array_size_user = QIB7322_EAGER_ARRAY_SIZE_18CTX_USER;
 		break;
 	default:
-		linker_assert ( 0, invalid_QIB7322_NUM_CONTEXTS );
+		build_assert ( 0 );
 		return -EINVAL;
 	}
 
@@ -1351,7 +1351,7 @@ static int qib7322_post_recv ( struct ib_device *ibdev,
 	case 16384: bufsize = QIB7322_EAGER_BUFFER_16K; break;
 	case 32768: bufsize = QIB7322_EAGER_BUFFER_32K; break;
 	case 65536: bufsize = QIB7322_EAGER_BUFFER_64K; break;
-	default:    linker_assert ( 0, invalid_rx_payload_size );
+	default:    build_assert ( 0 );
 		    bufsize = QIB7322_EAGER_BUFFER_NONE;
 	}
 
diff --git a/src/drivers/net/3c595.c b/src/drivers/net/3c595.c
index 92d38cfc..c6983100 100644
--- a/src/drivers/net/3c595.c
+++ b/src/drivers/net/3c595.c
@@ -523,10 +523,12 @@ static struct nic_operations t595_operations = {
 };
 
 static struct pci_device_id t595_nics[] = {
+PCI_ROM(0x10b7, 0x4500, "3c450-1",         "3Com450 HomePNA Tornado", 0),
 PCI_ROM(0x10b7, 0x5900, "3c590",           "3Com590", 0),		/* Vortex 10Mbps */
 PCI_ROM(0x10b7, 0x5950, "3c595",           "3Com595", 0),		/* Vortex 100baseTx */
 PCI_ROM(0x10b7, 0x5951, "3c595-1",         "3Com595", 0),		/* Vortex 100baseT4 */
 PCI_ROM(0x10b7, 0x5952, "3c595-2",         "3Com595", 0),		/* Vortex 100base-MII */
+PCI_ROM(0x10b7, 0x7646, "3csoho100-tx-1",  "3CSOHO100-TX", 0),	/* Hurricane */
 PCI_ROM(0x10b7, 0x9000, "3c900-tpo",       "3Com900-TPO", 0),	/* 10 Base TPO */
 PCI_ROM(0x10b7, 0x9001, "3c900-t4",        "3Com900-Combo", 0),	/* 10/100 T4 */
 PCI_ROM(0x10b7, 0x9004, "3c900b-tpo",      "3Com900B-TPO", 0),	/* 10 Base TPO */
@@ -535,8 +537,6 @@ PCI_ROM(0x10b7, 0x9006, "3c900b-tpb2",     "3Com900B-2/T", 0),	/* 10 Base TP and
 PCI_ROM(0x10b7, 0x900a, "3c900b-fl",       "3Com900B-FL", 0),	/* 10 Base F */
 PCI_ROM(0x10b7, 0x9800, "3c980-cyclone-1", "3Com980-Cyclone", 0),	/* Cyclone */
 PCI_ROM(0x10b7, 0x9805, "3c9805-1",        "3Com9805", 0),		/* Dual Port Server Cyclone */
-PCI_ROM(0x10b7, 0x7646, "3csoho100-tx-1",  "3CSOHO100-TX", 0),	/* Hurricane */
-PCI_ROM(0x10b7, 0x4500, "3c450-1",         "3Com450 HomePNA Tornado", 0),
 };
 
 PCI_DRIVER ( t595_driver, t595_nics, PCI_NO_CLASS );
diff --git a/src/drivers/net/3c90x.c b/src/drivers/net/3c90x.c
index 1b8190c4..a94473ef 100644
--- a/src/drivers/net/3c90x.c
+++ b/src/drivers/net/3c90x.c
@@ -955,16 +955,20 @@ static int a3c90x_probe(struct pci_device *pci)
 
 static struct pci_device_id a3c90x_nics[] = {
 /* Original 90x revisions: */
+	PCI_ROM(0x10b7, 0x1201, "3c982a", "3Com982A", 0),
+	PCI_ROM(0x10b7, 0x1202, "3c982b", "3Com982B", 0),
+	PCI_ROM(0x10b7, 0x4500, "3c450", "3Com450 HomePNA Tornado", 0),
 	PCI_ROM(0x10b7, 0x6055, "3c556", "3C556", 0),	/* Huricane */
+	PCI_ROM(0x10b7, 0x7646, "3csoho100-tx", "3CSOHO100-TX", 0),	/* Hurricane */
 	PCI_ROM(0x10b7, 0x9000, "3c905-tpo", "3Com900-TPO", 0),	/* 10 Base TPO */
 	PCI_ROM(0x10b7, 0x9001, "3c905-t4", "3Com900-Combo", 0),	/* 10/100 T4 */
-	PCI_ROM(0x10b7, 0x9050, "3c905-tpo100", "3Com905-TX", 0),	/* 100 Base TX / 10/100 TPO */
-	PCI_ROM(0x10b7, 0x9051, "3c905-combo", "3Com905-T4", 0),	/* 100 Base T4 / 10 Base Combo */
 /* Newer 90xB revisions: */
 	PCI_ROM(0x10b7, 0x9004, "3c905b-tpo", "3Com900B-TPO", 0),	/* 10 Base TPO */
 	PCI_ROM(0x10b7, 0x9005, "3c905b-combo", "3Com900B-Combo", 0),	/* 10 Base Combo */
 	PCI_ROM(0x10b7, 0x9006, "3c905b-tpb2", "3Com900B-2/T", 0),	/* 10 Base TP and Base2 */
 	PCI_ROM(0x10b7, 0x900a, "3c905b-fl", "3Com900B-FL", 0),	/* 10 Base FL */
+	PCI_ROM(0x10b7, 0x9050, "3c905-tpo100", "3Com905-TX", 0),	/* 100 Base TX / 10/100 TPO */
+	PCI_ROM(0x10b7, 0x9051, "3c905-combo", "3Com905-T4", 0),	/* 100 Base T4 / 10 Base Combo */
 	PCI_ROM(0x10b7, 0x9055, "3c905b-tpo100", "3Com905B-TX", 0),	/* 10/100 TPO */
 	PCI_ROM(0x10b7, 0x9056, "3c905b-t4", "3Com905B-T4", 0),	/* 10/100 T4 */
 	PCI_ROM(0x10b7, 0x9058, "3c905b-9058", "3Com905B-9058", 0),	/* Cyclone 10/100/BNC */
@@ -975,10 +979,6 @@ static struct pci_device_id a3c90x_nics[] = {
 	PCI_ROM(0x10b7, 0x9210, "3c920b-emb-wnm", "3Com20B-EMB WNM", 0),
 	PCI_ROM(0x10b7, 0x9800, "3c980", "3Com980-Cyclone", 0),	/* Cyclone */
 	PCI_ROM(0x10b7, 0x9805, "3c9805", "3Com9805", 0),	/* Dual Port Server Cyclone */
-	PCI_ROM(0x10b7, 0x7646, "3csoho100-tx", "3CSOHO100-TX", 0),	/* Hurricane */
-	PCI_ROM(0x10b7, 0x4500, "3c450", "3Com450 HomePNA Tornado", 0),
-	PCI_ROM(0x10b7, 0x1201, "3c982a", "3Com982A", 0),
-	PCI_ROM(0x10b7, 0x1202, "3c982b", "3Com982B", 0),
 };
 
 struct pci_driver a3c90x_driver __pci_driver = {
diff --git a/src/drivers/net/ath/ath5k/ath5k.c b/src/drivers/net/ath/ath5k/ath5k.c
index e43eb0aa..643884d4 100644
--- a/src/drivers/net/ath/ath5k/ath5k.c
+++ b/src/drivers/net/ath/ath5k/ath5k.c
@@ -65,14 +65,11 @@ FILE_LICENCE ( BSD3 );
 
 /* Known PCI ids */
 static struct pci_device_id ath5k_nics[] = {
-	PCI_ROM(0x168c, 0x0207, "ath5210e", "Atheros 5210 early", AR5K_AR5210),
+	PCI_ROM(0x10b7, 0x0013, "rdag675", "3com 3CRDAG675", AR5K_AR5212),
 	PCI_ROM(0x168c, 0x0007, "ath5210", "Atheros 5210", AR5K_AR5210),
 	PCI_ROM(0x168c, 0x0011, "ath5311", "Atheros 5311 (AHB)", AR5K_AR5211),
 	PCI_ROM(0x168c, 0x0012, "ath5211", "Atheros 5211", AR5K_AR5211),
 	PCI_ROM(0x168c, 0x0013, "ath5212", "Atheros 5212", AR5K_AR5212),
-	PCI_ROM(0xa727, 0x0013, "ath5212c","3com Ath 5212", AR5K_AR5212),
-	PCI_ROM(0x10b7, 0x0013, "rdag675", "3com 3CRDAG675", AR5K_AR5212),
-	PCI_ROM(0x168c, 0x1014, "ath5212m", "Ath 5212 miniPCI", AR5K_AR5212),
 	PCI_ROM(0x168c, 0x0014, "ath5212x14", "Atheros 5212 x14", AR5K_AR5212),
 	PCI_ROM(0x168c, 0x0015, "ath5212x15", "Atheros 5212 x15", AR5K_AR5212),
 	PCI_ROM(0x168c, 0x0016, "ath5212x16", "Atheros 5212 x16", AR5K_AR5212),
@@ -83,6 +80,9 @@ static struct pci_device_id ath5k_nics[] = {
 	PCI_ROM(0x168c, 0x001b, "ath5413", "Atheros 5413 Eagle", AR5K_AR5212),
 	PCI_ROM(0x168c, 0x001c, "ath5212e", "Atheros 5212 PCI-E", AR5K_AR5212),
 	PCI_ROM(0x168c, 0x001d, "ath2417", "Atheros 2417 Nala", AR5K_AR5212),
+	PCI_ROM(0x168c, 0x0207, "ath5210e", "Atheros 5210 early", AR5K_AR5210),
+	PCI_ROM(0x168c, 0x1014, "ath5212m", "Ath 5212 miniPCI", AR5K_AR5212),
+	PCI_ROM(0xa727, 0x0013, "ath5212c","3com Ath 5212", AR5K_AR5212),
 };
 
 #define ATH5K_SPMBL_NO   1
diff --git a/src/drivers/net/b44.c b/src/drivers/net/b44.c
index 1ca7e2e5..30ece557 100644
--- a/src/drivers/net/b44.c
+++ b/src/drivers/net/b44.c
@@ -945,8 +945,8 @@ static struct net_device_operations b44_operations = {
 
 
 static struct pci_device_id b44_nics[] = {
-	PCI_ROM(0x14e4, 0x4401, "BCM4401", "BCM4401", 0),
 	PCI_ROM(0x14e4, 0x170c, "BCM4401-B0", "BCM4401-B0", 0),
+	PCI_ROM(0x14e4, 0x4401, "BCM4401", "BCM4401", 0),
 	PCI_ROM(0x14e4, 0x4402, "BCM4401-B1", "BCM4401-B1", 0),
 };
 
diff --git a/src/drivers/net/bnxt/bnxt.c b/src/drivers/net/bnxt/bnxt.c
index 605aea32..a127f6ce 100644
--- a/src/drivers/net/bnxt/bnxt.c
+++ b/src/drivers/net/bnxt/bnxt.c
@@ -24,6 +24,11 @@ static int bnxt_rx_complete ( struct net_device *dev, struct rx_pkt_cmpl *rx );
 void bnxt_link_evt ( struct bnxt *bp, struct hwrm_async_event_cmpl *evt );
 
 static struct pci_device_id bnxt_nics[] = {
+	PCI_ROM( 0x14e4, 0x1604, "14e4-1604", "14e4-1604", 0 ),
+	PCI_ROM( 0x14e4, 0x1605, "14e4-1605", "14e4-1605", 0 ),
+	PCI_ROM( 0x14e4, 0x1606, "14e4-1606", "14e4-1606", 0 ),
+	PCI_ROM( 0x14e4, 0x1609, "14e4-1609", "14e4-1609", 0 ),
+	PCI_ROM( 0x14e4, 0x1614, "14e4-1614", "14e4-1614", 0 ),
 	PCI_ROM( 0x14e4, 0x16c0, "14e4-16C0", "14e4-16C0", 0 ),
 	PCI_ROM( 0x14e4, 0x16c1, "14e4-16C1", "14e4-16C1", BNXT_FLAG_PCI_VF ),
 	PCI_ROM( 0x14e4, 0x16c8, "14e4-16C8", "14e4-16C8", 0 ),
@@ -62,26 +67,22 @@ static struct pci_device_id bnxt_nics[] = {
 	PCI_ROM( 0x14e4, 0x16ef, "14e4-16EF", "14e4-16EF", 0 ),
 	PCI_ROM( 0x14e4, 0x16f0, "14e4-16F0", "14e4-16F0", 0 ),
 	PCI_ROM( 0x14e4, 0x16f1, "14e4-16F1", "14e4-16F1", 0 ),
-	PCI_ROM( 0x14e4, 0x1604, "14e4-1604", "14e4-1604", 0 ),
-	PCI_ROM( 0x14e4, 0x1605, "14e4-1605", "14e4-1605", 0 ),
-	PCI_ROM( 0x14e4, 0x1606, "14e4-1606", "14e4-1606", 0 ),
-	PCI_ROM( 0x14e4, 0x1609, "14e4-1609", "14e4-1609", 0 ),
-	PCI_ROM( 0x14e4, 0x1614, "14e4-1614", "14e4-1614", 0 ),
-	PCI_ROM( 0x14e4, 0xd802, "14e4-D802", "14e4-D802", 0 ),
-	PCI_ROM( 0x14e4, 0xd804, "14e4-D804", "14e4-D804", 0 ),
 	PCI_ROM( 0x14e4, 0x1750, "14e4-1750", "14e4-1750", 0 ),
-	PCI_ROM( 0x14e4, 0x1802, "14e4-1802", "14e4-1802", 0 ),
-	PCI_ROM( 0x14e4, 0x1805, "14e4-1805", "14e4-1805", 0 ),
 	PCI_ROM( 0x14e4, 0x1751, "14e4-1751", "14e4-1751", 0 ),
-	PCI_ROM( 0x14e4, 0x1801, "14e4-1801", "14e4-1801", 0 ),
-	PCI_ROM( 0x14e4, 0x1804, "14e4-1804", "14e4-1804", 0 ),
 	PCI_ROM( 0x14e4, 0x1752, "14e4-1752", "14e4-1752", 0 ),
+	PCI_ROM( 0x14e4, 0x1760, "14e4-1760", "14e4-1760", 0 ),
 	PCI_ROM( 0x14e4, 0x1800, "14e4-1800", "14e4-1800", 0 ),
+	PCI_ROM( 0x14e4, 0x1801, "14e4-1801", "14e4-1801", 0 ),
+	PCI_ROM( 0x14e4, 0x1802, "14e4-1802", "14e4-1802", 0 ),
 	PCI_ROM( 0x14e4, 0x1803, "14e4-1803", "14e4-1803", 0 ),
+	PCI_ROM( 0x14e4, 0x1804, "14e4-1804", "14e4-1804", 0 ),
+	PCI_ROM( 0x14e4, 0x1805, "14e4-1805", "14e4-1805", 0 ),
 	PCI_ROM( 0x14e4, 0x1806, "14e4-1806", "14e4-1806", BNXT_FLAG_PCI_VF ),
 	PCI_ROM( 0x14e4, 0x1807, "14e4-1807", "14e4-1807", BNXT_FLAG_PCI_VF ),
 	PCI_ROM( 0x14e4, 0x1808, "14e4-1808", "14e4-1808", BNXT_FLAG_PCI_VF ),
 	PCI_ROM( 0x14e4, 0x1809, "14e4-1809", "14e4-1809", BNXT_FLAG_PCI_VF ),
+	PCI_ROM( 0x14e4, 0xd802, "14e4-D802", "14e4-D802", 0 ),
+	PCI_ROM( 0x14e4, 0xd804, "14e4-D804", "14e4-D804", 0 ),
 };
 
 /**
@@ -181,7 +182,7 @@ static void bnxt_set_link ( struct bnxt *bp )
 		netdev_link_down ( bp->dev );
 }
 
-static void thor_db ( struct bnxt *bp, u32 idx, u32 xid, u32 flag )
+static void dev_p5_db ( struct bnxt *bp, u32 idx, u32 xid, u32 flag )
 {
 	void *off;
 	u64  val;
@@ -196,10 +197,28 @@ static void thor_db ( struct bnxt *bp, u32 idx, u32 xid, u32 flag )
 	write64 ( val, off );
 }
 
+static void dev_p7_db ( struct bnxt *bp, u32 idx, u32 xid, u32 flag, u32 epoch, u32 toggle )
+{
+	void *off;
+	u64  val;
+
+	off = ( void * ) ( bp->bar1 );
+
+	val = ( ( u64 )DBC_MSG_XID ( xid, flag ) << 32 ) |
+	        ( u64 )DBC_MSG_IDX ( idx ) |
+	        ( u64 )DBC_MSG_EPCH ( epoch ) |
+	        ( u64 )DBC_MSG_TOGGLE ( toggle );
+	write64 ( val, off );
+}
+
 static void bnxt_db_nq ( struct bnxt *bp )
 {
-	if ( bp->thor )
-		thor_db ( bp, ( u32 )bp->nq.cons_id,
+	if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P7 ) )
+		dev_p7_db ( bp, ( u32 )bp->nq.cons_id,
+			 ( u32 )bp->nq_ring_id, DBC_DBC_TYPE_NQ_ARM,
+                         ( u32 )bp->nq.epoch, 0 );
+	else if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5 ) )
+		dev_p5_db ( bp, ( u32 )bp->nq.cons_id,
 			 ( u32 )bp->nq_ring_id, DBC_DBC_TYPE_NQ_ARM );
 	else
 		write32 ( CMPL_DOORBELL_KEY_CMPL, ( bp->bar1 + 0 ) );
@@ -207,8 +226,12 @@ static void bnxt_db_nq ( struct bnxt *bp )
 
 static void bnxt_db_cq ( struct bnxt *bp )
 {
-	if ( bp->thor )
-		thor_db ( bp, ( u32 )bp->cq.cons_id,
+	if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P7 ) )
+		dev_p7_db ( bp, ( u32 )bp->cq.cons_id,
+			 ( u32 )bp->cq_ring_id, DBC_DBC_TYPE_CQ_ARMALL,
+                         ( u32 )bp->cq.epoch, ( u32 )bp->nq.toggle );
+	else if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5 ) )
+		dev_p5_db ( bp, ( u32 )bp->cq.cons_id,
 			 ( u32 )bp->cq_ring_id, DBC_DBC_TYPE_CQ_ARMALL );
 	else
 		write32 ( CQ_DOORBELL_KEY_IDX ( bp->cq.cons_id ),
@@ -217,16 +240,22 @@ static void bnxt_db_cq ( struct bnxt *bp )
 
 static void bnxt_db_rx ( struct bnxt *bp, u32 idx )
 {
-	if ( bp->thor )
-		thor_db ( bp, idx, ( u32 )bp->rx_ring_id, DBC_DBC_TYPE_SRQ );
+	if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P7 ) )
+		dev_p7_db ( bp, idx, ( u32 )bp->rx_ring_id, DBC_DBC_TYPE_SRQ,
+                         ( u32 )bp->rx.epoch, 0 );
+	else if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5 ) )
+		dev_p5_db ( bp, idx, ( u32 )bp->rx_ring_id, DBC_DBC_TYPE_SRQ );
 	else
 		write32 ( RX_DOORBELL_KEY_RX | idx, ( bp->bar1 + 0 ) );
 }
 
 static void bnxt_db_tx ( struct bnxt *bp, u32 idx )
 {
-	if ( bp->thor )
-		thor_db ( bp, idx, ( u32 )bp->tx_ring_id, DBC_DBC_TYPE_SQ );
+	if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P7 ) )
+		dev_p7_db ( bp, idx, ( u32 )bp->tx_ring_id, DBC_DBC_TYPE_SQ,
+                         ( u32 )bp->tx.epoch, 0 );
+	else if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5 ) )
+		dev_p5_db ( bp, idx, ( u32 )bp->tx_ring_id, DBC_DBC_TYPE_SQ );
 	else
 		write32 ( ( u32 ) ( TX_DOORBELL_KEY_TX | idx ),
 			( bp->bar1 + 0 ) );
@@ -253,6 +282,31 @@ static u16 bnxt_get_pkt_vlan ( char *src )
 	return 0;
 }
 
+static u16 bnxt_get_rx_vlan ( struct rx_pkt_cmpl *rx_cmp, struct rx_pkt_cmpl_hi *rx_cmp_hi )
+{
+	struct rx_pkt_v3_cmpl *rx_cmp_v3 = ( struct rx_pkt_v3_cmpl * )rx_cmp;
+	struct rx_pkt_v3_cmpl_hi *rx_cmp_hi_v3 = ( struct rx_pkt_v3_cmpl_hi * )rx_cmp_hi;
+	u16 rx_vlan;
+
+	/* Get VLAN ID from RX completion ring */
+	if ( ( rx_cmp_v3->flags_type & RX_PKT_V3_CMPL_TYPE_MASK ) ==
+	     RX_PKT_V3_CMPL_TYPE_RX_L2_V3 ) {
+		if ( rx_cmp_hi_v3->flags2 & RX_PKT_V3_CMPL_HI_FLAGS2_META_FORMAT_ACT_REC_PTR )
+			rx_vlan = ( rx_cmp_hi_v3->metadata0 &
+				RX_PKT_V3_CMPL_HI_METADATA0_VID_MASK );
+		else
+			rx_vlan = 0;
+	} else {
+	        if ( rx_cmp_hi->flags2 & RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN )
+			rx_vlan = ( rx_cmp_hi->metadata &
+				RX_PKT_CMPL_METADATA_VID_MASK );
+	        else
+			rx_vlan = 0;
+	}
+
+	return rx_vlan;
+}
+
 int bnxt_vlan_drop ( struct bnxt *bp, u16 rx_vlan )
 {
 	if ( rx_vlan ) {
@@ -382,6 +436,9 @@ int bnxt_post_rx_buffers ( struct bnxt *bp )
 			}
 		}
 		cons_id = NEXT_IDX ( cons_id, bp->rx.ring_cnt );
+		/* If the ring has wrapped, flip the epoch bit */
+		if ( iob_idx > cons_id )
+		        bp->rx.epoch ^= 1;
 		bp->rx.iob_cnt++;
 	}
 
@@ -396,14 +453,21 @@ int bnxt_post_rx_buffers ( struct bnxt *bp )
 }
 
 u8 bnxt_rx_drop ( struct bnxt *bp, struct io_buffer *iob,
+	struct rx_pkt_cmpl *rx_cmp,
 	struct rx_pkt_cmpl_hi *rx_cmp_hi, u16 rx_len )
 {
+	struct rx_pkt_v3_cmpl *rx_cmp_v3 = ( struct rx_pkt_v3_cmpl * )rx_cmp;
+	struct rx_pkt_v3_cmpl_hi  *rx_cmp_hi_v3 = ( struct rx_pkt_v3_cmpl_hi * )rx_cmp_hi;
 	u8  *rx_buf = ( u8 * )iob->data;
 	u16 err_flags, rx_vlan;
 	u8  ignore_chksum_err = 0;
 	int i;
 
-	err_flags = rx_cmp_hi->errors_v2 >> RX_PKT_CMPL_ERRORS_BUFFER_ERROR_SFT;
+	if ( ( rx_cmp_v3->flags_type & RX_PKT_V3_CMPL_TYPE_MASK ) ==
+	     RX_PKT_V3_CMPL_TYPE_RX_L2_V3 ) {
+	        err_flags = rx_cmp_hi_v3->errors_v2 >> RX_PKT_V3_CMPL_HI_ERRORS_BUFFER_ERROR_SFT;
+	} else
+	        err_flags = rx_cmp_hi->errors_v2 >> RX_PKT_CMPL_ERRORS_BUFFER_ERROR_SFT;
 	if ( rx_cmp_hi->errors_v2 == 0x20 || rx_cmp_hi->errors_v2 == 0x21 )
 		ignore_chksum_err = 1;
 
@@ -423,13 +487,7 @@ u8 bnxt_rx_drop ( struct bnxt *bp, struct io_buffer *iob,
 		return 2;
 	}
 
-	/* Get VLAN ID from RX completion ring */
-	if ( rx_cmp_hi->flags2 & RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN )
-		rx_vlan = ( rx_cmp_hi->metadata &
-				RX_PKT_CMPL_METADATA_VID_MASK );
-	else
-		rx_vlan = 0;
-
+	rx_vlan = bnxt_get_rx_vlan ( rx_cmp, rx_cmp_hi );
 	dbg_rx_vlan ( bp, rx_cmp_hi->metadata, rx_cmp_hi->flags2, rx_vlan );
 	if ( bnxt_vlan_drop ( bp, rx_vlan ) ) {
 		bp->rx.drop_vlan++;
@@ -449,10 +507,11 @@ static void bnxt_adv_cq_index ( struct bnxt *bp, u16 cnt )
 	u16 cons_id;
 
 	cons_id = bp->cq.cons_id + cnt;
-	if ( cons_id >= MAX_CQ_DESC_CNT ) {
+	if ( cons_id >= bp->cq.ring_cnt) {
 		/* Toggle completion bit when the ring wraps. */
 		bp->cq.completion_bit ^= 1;
-		cons_id = cons_id - MAX_CQ_DESC_CNT;
+		bp->cq.epoch ^= 1;
+		cons_id = cons_id - bp->cq.ring_cnt;
 	}
 	bp->cq.cons_id = cons_id;
 }
@@ -466,7 +525,7 @@ void bnxt_rx_process ( struct net_device *dev, struct bnxt *bp,
 
 	dump_rx_bd ( rx_cmp, rx_cmp_hi, desc_idx );
 	assert ( !iob );
-	drop = bnxt_rx_drop ( bp, iob, rx_cmp_hi, rx_cmp->len );
+	drop = bnxt_rx_drop ( bp, iob, rx_cmp, rx_cmp_hi, rx_cmp->len );
 	dbg_rxp ( iob->data, rx_cmp->len, drop );
 	if ( drop )
 		netdev_rx_err ( dev, iob, -EINVAL );
@@ -531,12 +590,17 @@ void bnxt_mm_nic ( struct bnxt *bp )
 	memset ( bp->nq.bd_virt, 0, NQ_RING_BUFFER_SIZE );
 	bp->nq.cons_id		= 0;
 	bp->nq.completion_bit	= 0x1;
+	bp->nq.epoch            = 0;
+	bp->nq.toggle           = 0;
 	bp->cq.cons_id		= 0;
 	bp->cq.completion_bit	= 0x1;
+	bp->cq.epoch            = 0;
 	bp->tx.prod_id		= 0;
 	bp->tx.cons_id		= 0;
+	bp->tx.epoch            = 0;
 	bp->rx.cons_id		= 0;
 	bp->rx.iob_cnt		= 0;
+	bp->rx.epoch            = 0;
 
 	bp->link_status		= STATUS_LINK_DOWN;
 	bp->wait_link_timeout	= LINK_DEFAULT_TIMEOUT;
@@ -722,8 +786,16 @@ static int bnxt_hwrm_ver_get ( struct bnxt *bp )
 		 ( resp->dev_caps_cfg & SHORT_CMD_REQUIRED ) )
 		FLAG_SET ( bp->flags, BNXT_FLAG_HWRM_SHORT_CMD_SUPP );
 	bp->hwrm_max_ext_req_len = resp->max_ext_req_len;
-	if ( bp->chip_num == CHIP_NUM_57500 )
-		bp->thor = 1;
+	if ( ( bp->chip_num == CHIP_NUM_57508 ) ||
+	     ( bp->chip_num == CHIP_NUM_57504 ) ||
+	     ( bp->chip_num == CHIP_NUM_57502 ) ) {
+		FLAG_SET ( bp->flags, BNXT_FLAG_IS_CHIP_P5 );
+		FLAG_SET ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS );
+	}
+	if ( bp->chip_num == CHIP_NUM_57608 ) {
+		FLAG_SET ( bp->flags, BNXT_FLAG_IS_CHIP_P7 );
+		FLAG_SET ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS );
+	}
 	dbg_fw_ver ( resp, bp->hwrm_cmd_timeout );
 	return STATUS_SUCCESS;
 }
@@ -915,6 +987,30 @@ static int bnxt_hwrm_func_qcfg_req ( struct bnxt *bp )
 	return STATUS_SUCCESS;
 }
 
+static int bnxt_hwrm_port_phy_qcaps_req ( struct bnxt *bp )
+{
+	u16 cmd_len = ( u16 )sizeof ( struct hwrm_port_phy_qcaps_input );
+	struct hwrm_port_phy_qcaps_input *req;
+	struct hwrm_port_phy_qcaps_output *resp;
+	int rc;
+
+	DBGP ( "%s\n", __func__ );
+
+	req = ( struct hwrm_port_phy_qcaps_input * )bp->hwrm_addr_req;
+	resp = ( struct hwrm_port_phy_qcaps_output * )bp->hwrm_addr_resp;
+	hwrm_init ( bp, ( void * )req, ( u16 )HWRM_PORT_PHY_QCAPS, cmd_len );
+	rc = wait_resp ( bp, bp->hwrm_cmd_timeout, cmd_len, __func__ );
+	if ( rc ) {
+		DBGP ( "-s %s (  ): Failed\n", __func__ );
+		return STATUS_FAILURE;
+	}
+
+	if ( resp->flags2 & PORT_PHY_QCAPS_RESP_FLAGS2_SPEEDS2_SUPPORTED )
+                FLAG_SET ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 );
+
+	return STATUS_SUCCESS;
+}
+
 static int bnxt_hwrm_func_reset_req ( struct bnxt *bp )
 {
 	u16 cmd_len = ( u16 )sizeof ( struct hwrm_func_reset_input );
@@ -942,7 +1038,7 @@ static int bnxt_hwrm_func_cfg_req ( struct bnxt *bp )
 	hwrm_init ( bp, ( void * )req, ( u16 )HWRM_FUNC_CFG, cmd_len );
 	req->fid = ( u16 )HWRM_NA_SIGNATURE;
 	bnxt_hwrm_assign_resources ( bp );
-	if ( bp->thor ) {
+	if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) ) {
 		req->enables |= ( FUNC_CFG_REQ_ENABLES_NUM_MSIX |
 				  FUNC_CFG_REQ_ENABLES_NUM_VNICS |
 				  FUNC_CFG_REQ_ENABLES_EVB_MODE );
@@ -1009,7 +1105,7 @@ static int bnxt_hwrm_set_async_event ( struct bnxt *bp )
 	u16 idx;
 
 	DBGP ( "%s\n", __func__ );
-	if ( bp->thor )
+	if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) )
 		idx = bp->nq_ring_id;
 	else
 		idx = bp->cq_ring_id;
@@ -1160,6 +1256,10 @@ static int bnxt_hwrm_port_phy_qcfg ( struct bnxt *bp, u16 idx )
 	if ( idx & SUPPORT_SPEEDS )
 		bp->support_speeds = resp->support_speeds;
 
+	if ( idx & SUPPORT_SPEEDS2 )
+                if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) )
+		        bp->auto_link_speeds2_mask = resp->auto_link_speeds2;
+
 	if ( idx & DETECT_MEDIA )
 		bp->media_detect = resp->module_status;
 
@@ -1199,22 +1299,24 @@ static int bnxt_get_link_speed ( struct bnxt *bp )
 	u32 *ptr32 = ( u32 * )bp->hwrm_addr_dma;
 
 	DBGP ( "%s\n", __func__ );
-	test_if ( bnxt_hwrm_nvm_get_variable_req ( bp, 4,
-		( u16 )LINK_SPEED_DRV_NUM,
-		1, ( u16 )bp->port_idx ) != STATUS_SUCCESS )
-		return STATUS_FAILURE;
-	bp->link_set = SET_LINK ( *ptr32, SPEED_DRV_MASK, SPEED_DRV_SHIFT );
+	if ( ! ( FLAG_TEST (bp->flags, BNXT_FLAG_IS_CHIP_P7 ) ) ) {
+	        test_if ( bnxt_hwrm_nvm_get_variable_req ( bp, 4,
+		        ( u16 )LINK_SPEED_DRV_NUM,
+		        1, ( u16 )bp->port_idx ) != STATUS_SUCCESS )
+		        return STATUS_FAILURE;
+	        bp->link_set = SET_LINK ( *ptr32, SPEED_DRV_MASK, SPEED_DRV_SHIFT );
+	        test_if ( bnxt_hwrm_nvm_get_variable_req ( bp, 4,
+		        ( u16 )D3_LINK_SPEED_FW_NUM, 1,
+		        ( u16 )bp->port_idx ) != STATUS_SUCCESS )
+		        return STATUS_FAILURE;
+	        bp->link_set |= SET_LINK ( *ptr32, D3_SPEED_FW_MASK,
+				D3_SPEED_FW_SHIFT );
+	}
 	test_if ( bnxt_hwrm_nvm_get_variable_req ( bp, 4,
 		( u16 )LINK_SPEED_FW_NUM,
 		1, ( u16 )bp->port_idx ) != STATUS_SUCCESS )
 		return STATUS_FAILURE;
 	bp->link_set |= SET_LINK ( *ptr32, SPEED_FW_MASK, SPEED_FW_SHIFT );
-	test_if ( bnxt_hwrm_nvm_get_variable_req ( bp, 4,
-		 ( u16 )D3_LINK_SPEED_FW_NUM, 1,
-		 ( u16 )bp->port_idx ) != STATUS_SUCCESS )
-		return STATUS_FAILURE;
-	bp->link_set |= SET_LINK ( *ptr32, D3_SPEED_FW_MASK,
-			D3_SPEED_FW_SHIFT );
 	test_if ( bnxt_hwrm_nvm_get_variable_req ( bp, 1,
 		 ( u16 )PORT_CFG_LINK_SETTINGS_MEDIA_AUTO_DETECT_NUM,
 		1, ( u16 )bp->port_idx ) != STATUS_SUCCESS )
@@ -1222,32 +1324,51 @@ static int bnxt_get_link_speed ( struct bnxt *bp )
 	bp->link_set |= SET_LINK ( *ptr32,
 		MEDIA_AUTO_DETECT_MASK, MEDIA_AUTO_DETECT_SHIFT );
 
-	switch ( bp->link_set & LINK_SPEED_DRV_MASK ) {
-	case LINK_SPEED_DRV_1G:
+	/* Use LINK_SPEED_FW_xxx which is valid for CHIP_P7 and earlier devices */
+	switch ( bp->link_set & LINK_SPEED_FW_MASK ) {
+	case LINK_SPEED_FW_1G:
 		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_1000MBPS );
 		break;
-	case LINK_SPEED_DRV_2_5G:
+	case LINK_SPEED_FW_2_5G:
 		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_2500MBPS );
 		break;
-	case LINK_SPEED_DRV_10G:
+	case LINK_SPEED_FW_10G:
 		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_10GBPS );
 		break;
-	case LINK_SPEED_DRV_25G:
+	case LINK_SPEED_FW_25G:
 		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_25GBPS );
 		break;
-	case LINK_SPEED_DRV_40G:
+	case LINK_SPEED_FW_40G:
 		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_40GBPS );
 		break;
-	case LINK_SPEED_DRV_50G:
+	case LINK_SPEED_FW_50G:
 		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_50GBPS );
 		break;
-	case LINK_SPEED_DRV_100G:
+	case LINK_SPEED_FW_50G_PAM4:
+		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_50PAM4GBPS );
+		break;
+	case LINK_SPEED_FW_100G:
 		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_100GBPS );
 		break;
-	case LINK_SPEED_DRV_200G:
+	case LINK_SPEED_FW_100G_PAM4:
+		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_100PAM4GBPS );
+		break;
+	case LINK_SPEED_FW_100G_PAM4_112:
+		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_100PAM4_112GBPS );
+		break;
+	case LINK_SPEED_FW_200G:
 		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_200GBPS );
 		break;
-	case LINK_SPEED_DRV_AUTONEG:
+	case LINK_SPEED_FW_200G_PAM4_112:
+		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_200PAM4_112GBPS );
+		break;
+	case LINK_SPEED_FW_400G_PAM4:
+		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_400PAM4GBPS );
+		break;
+	case LINK_SPEED_FW_400G_PAM4_112:
+		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_400PAM4_112GBPS );
+		break;
+	case LINK_SPEED_FW_AUTONEG:
 		bp->medium = SET_MEDIUM_SPEED ( bp, MEDIUM_SPEED_AUTONEG );
 		break;
 	default:
@@ -1266,27 +1387,29 @@ static int bnxt_get_vlan ( struct bnxt *bp )
 	if ( bp->vf )
 		return STATUS_SUCCESS;
 
-	test_if ( bnxt_hwrm_nvm_get_variable_req ( bp, 1,
-		 ( u16 )FUNC_CFG_PRE_BOOT_MBA_VLAN_NUM, 1,
-		 ( u16 )bp->ordinal_value ) != STATUS_SUCCESS )
-		return STATUS_FAILURE;
+	if ( ! ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P7 ) ) ) {
+		test_if ( bnxt_hwrm_nvm_get_variable_req ( bp, 1,
+			 ( u16 )FUNC_CFG_PRE_BOOT_MBA_VLAN_NUM, 1,
+			 ( u16 )bp->ordinal_value ) != STATUS_SUCCESS )
+			return STATUS_FAILURE;
 
-	bp->mba_cfg2 = SET_MBA ( *ptr32, VLAN_MASK, VLAN_SHIFT );
-	test_if ( bnxt_hwrm_nvm_get_variable_req ( bp, 16,
-		 ( u16 )FUNC_CFG_PRE_BOOT_MBA_VLAN_VALUE_NUM, 1,
-		 ( u16 )bp->ordinal_value ) != STATUS_SUCCESS )
-		return STATUS_FAILURE;
+		bp->mba_cfg2 = SET_MBA ( *ptr32, VLAN_MASK, VLAN_SHIFT );
+		test_if ( bnxt_hwrm_nvm_get_variable_req ( bp, 16,
+			 ( u16 )FUNC_CFG_PRE_BOOT_MBA_VLAN_VALUE_NUM, 1,
+			 ( u16 )bp->ordinal_value ) != STATUS_SUCCESS )
+			return STATUS_FAILURE;
 
-	bp->mba_cfg2 |= SET_MBA ( *ptr32, VLAN_VALUE_MASK, VLAN_VALUE_SHIFT );
-	if ( bp->mba_cfg2 & FUNC_CFG_PRE_BOOT_MBA_VLAN_ENABLED )
-		bp->vlan_id = bp->mba_cfg2 & VLAN_VALUE_MASK;
-	else
-		bp->vlan_id = 0;
+		bp->mba_cfg2 |= SET_MBA ( *ptr32, VLAN_VALUE_MASK, VLAN_VALUE_SHIFT );
+		if ( bp->mba_cfg2 & FUNC_CFG_PRE_BOOT_MBA_VLAN_ENABLED )
+			bp->vlan_id = bp->mba_cfg2 & VLAN_VALUE_MASK;
+		else
+			bp->vlan_id = 0;
 
-	if ( bp->mba_cfg2 & FUNC_CFG_PRE_BOOT_MBA_VLAN_ENABLED )
-		DBGP ( "VLAN MBA Enabled ( %d )\n",
-			( bp->mba_cfg2 & VLAN_VALUE_MASK ) );
+		if ( bp->mba_cfg2 & FUNC_CFG_PRE_BOOT_MBA_VLAN_ENABLED )
+			DBGP ( "VLAN MBA Enabled ( %d )\n",
+				( bp->mba_cfg2 & VLAN_VALUE_MASK ) );
 
+	}
 	return STATUS_SUCCESS;
 }
 
@@ -1296,7 +1419,7 @@ static int bnxt_hwrm_backing_store_qcfg ( struct bnxt *bp )
 	struct hwrm_func_backing_store_qcfg_input *req;
 
 	DBGP ( "%s\n", __func__ );
-	if ( !bp->thor )
+	if ( ! ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) ) )
 		return STATUS_SUCCESS;
 
 	req = ( struct hwrm_func_backing_store_qcfg_input * )bp->hwrm_addr_req;
@@ -1311,7 +1434,7 @@ static int bnxt_hwrm_backing_store_cfg ( struct bnxt *bp )
 	struct hwrm_func_backing_store_cfg_input *req;
 
 	DBGP ( "%s\n", __func__ );
-	if ( !bp->thor )
+	if ( ! ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) ) )
 		return STATUS_SUCCESS;
 
 	req = ( struct hwrm_func_backing_store_cfg_input * )bp->hwrm_addr_req;
@@ -1330,7 +1453,7 @@ static int bnxt_hwrm_queue_qportcfg ( struct bnxt *bp )
 	int rc;
 
 	DBGP ( "%s\n", __func__ );
-	if ( !bp->thor )
+	if ( ! ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) ) )
 		return STATUS_SUCCESS;
 
 	req = ( struct hwrm_queue_qportcfg_input * )bp->hwrm_addr_req;
@@ -1370,7 +1493,10 @@ static int bnxt_hwrm_port_phy_cfg ( struct bnxt *bp )
 	u32 flags;
 	u32 enables = 0;
 	u16 force_link_speed = 0;
+	u16 force_link_speeds2 = 0;
+	u16 force_pam4_link_speed = 0;
 	u16 auto_link_speed_mask = 0;
+	u16 auto_link_speeds2_mask = 0;
 	u8  auto_mode = 0;
 	u8  auto_pause = 0;
 	u8  auto_duplex = 0;
@@ -1385,34 +1511,111 @@ static int bnxt_hwrm_port_phy_cfg ( struct bnxt *bp )
 		force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_1GB;
 		break;
 	case MEDIUM_SPEED_10GBPS:
-		force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_10GB;
+		if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+			force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_10GB;
+			enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+		} else {
+		        force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_10GB;
+		}
 		break;
 	case MEDIUM_SPEED_25GBPS:
-		force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_25GB;
+		if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+			force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_25GB;
+			enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+		} else {
+		        force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_25GB;
+		}
 		break;
 	case MEDIUM_SPEED_40GBPS:
-		force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_40GB;
+		if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+			force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_40GB;
+			enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+		} else {
+		        force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_40GB;
+		}
 		break;
 	case MEDIUM_SPEED_50GBPS:
-		force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_50GB;
+		if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+			force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_50GB;
+			enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+		} else {
+		        force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_50GB;
+		}
+		break;
+	case MEDIUM_SPEED_50PAM4GBPS:
+		if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+			force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_50GB_PAM4_56;
+			enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+		} else {
+		        force_pam4_link_speed = PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_50GB;
+			enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED;
+		}
 		break;
 	case MEDIUM_SPEED_100GBPS:
-		force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_100GB;
+		if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+			force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_100GB;
+			enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+		} else {
+		        force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_100GB;
+		}
+		break;
+	case MEDIUM_SPEED_100PAM4GBPS:
+		if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+			force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_100GB_PAM4_56;
+			enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+		} else {
+		        force_pam4_link_speed = PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_100GB;
+                        enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED;
+		}
+		break;
+	case MEDIUM_SPEED_100PAM4_112GBPS:
+		if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+			force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_100GB_PAM4_112;
+			enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+		}
 		break;
 	case MEDIUM_SPEED_200GBPS:
-		force_link_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_200GB;
+		if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+			force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_200GB_PAM4_56;
+			enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+		} else {
+			force_pam4_link_speed = PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_200GB;
+			enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED;
+		}
+		break;
+	case MEDIUM_SPEED_200PAM4_112GBPS:
+                if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+                        force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_200GB_PAM4_112;
+                        enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+                }
+                break;
+	case MEDIUM_SPEED_400PAM4GBPS:
+                if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+                        force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_400GB_PAM4_56;
+                        enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+                }
 		break;
+	case MEDIUM_SPEED_400PAM4_112GBPS:
+                if ( FLAG_TEST ( bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) ) {
+                        force_link_speeds2 = PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_400GB_PAM4_112;
+                        enables |= PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2;
+                }
+                break;
 	default:
 		auto_mode = PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
 		flags &= ~PORT_PHY_CFG_REQ_FLAGS_FORCE;
 		enables |= PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE |
-			PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK |
 			PORT_PHY_CFG_REQ_ENABLES_AUTO_DUPLEX |
 			PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE;
+		if ( FLAG_TEST (bp->flags, BNXT_FLAG_LINK_SPEEDS2 ) )
+			enables |= PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEEDS2_MASK;
+		else
+			enables |= PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK;
 		auto_pause = PORT_PHY_CFG_REQ_AUTO_PAUSE_TX |
 				PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
 		auto_duplex = PORT_PHY_CFG_REQ_AUTO_DUPLEX_BOTH;
 		auto_link_speed_mask = bp->support_speeds;
+		auto_link_speeds2_mask = bp->auto_link_speeds2_mask;
 		break;
 	}
 
@@ -1421,10 +1624,13 @@ static int bnxt_hwrm_port_phy_cfg ( struct bnxt *bp )
 	req->enables = enables;
 	req->port_id = bp->port_idx;
 	req->force_link_speed = force_link_speed;
+	req->force_pam4_link_speed = force_pam4_link_speed;
+	req->force_link_speeds2 = force_link_speeds2;
 	req->auto_mode = auto_mode;
 	req->auto_duplex = auto_duplex;
 	req->auto_pause = auto_pause;
 	req->auto_link_speed_mask = auto_link_speed_mask;
+	req->auto_link_speeds2_mask = auto_link_speeds2_mask;
 
 	return wait_resp ( bp, bp->hwrm_cmd_timeout, cmd_len, __func__ );
 }
@@ -1563,7 +1769,7 @@ static int bnxt_hwrm_ring_alloc_grp ( struct bnxt *bp )
 	int rc;
 
 	DBGP ( "%s\n", __func__ );
-	if ( bp->thor )
+	if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) )
 		return STATUS_SUCCESS;
 
 	req = ( struct hwrm_ring_grp_alloc_input * )bp->hwrm_addr_req;
@@ -1614,7 +1820,7 @@ static int bnxt_hwrm_ring_alloc ( struct bnxt *bp, u8 type )
 	switch ( type ) {
 	case RING_ALLOC_REQ_RING_TYPE_NQ:
 		req->page_size  = LM_PAGE_BITS ( 12 );
-		req->int_mode   = BNXT_CQ_INTR_MODE ( bp->vf );
+		req->int_mode   = BNXT_CQ_INTR_MODE ( ( (FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P7) ) || bp->vf ) );
 		req->length 	= ( u32 )bp->nq.ring_cnt;
 		req->logical_id = 0xFFFF; // Required value for Thor FW?
 		req->page_tbl_addr = virt_to_bus ( bp->nq.bd_virt );
@@ -1624,7 +1830,7 @@ static int bnxt_hwrm_ring_alloc ( struct bnxt *bp, u8 type )
 		req->int_mode  = BNXT_CQ_INTR_MODE ( bp->vf );
 		req->length    = ( u32 )bp->cq.ring_cnt;
 		req->page_tbl_addr = virt_to_bus ( bp->cq.bd_virt );
-		if ( !bp->thor )
+	        if ( ! ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) ) )
 			break;
 		req->enables = RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID;
 		req->nq_ring_id = bp->nq_ring_id;
@@ -1646,7 +1852,7 @@ static int bnxt_hwrm_ring_alloc ( struct bnxt *bp, u8 type )
 		req->stat_ctx_id = ( u32 )STAT_CTX_ID;
 		req->cmpl_ring_id  = bp->cq_ring_id;
 		req->page_tbl_addr = virt_to_bus ( bp->rx.bd_virt );
-		if ( !bp->thor )
+	        if ( ! ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) ) )
 			break;
 		req->queue_id    = ( u16 )RX_RING_QID;
 		req->rx_buf_size = MAX_ETHERNET_PACKET_BUFFER_SIZE;
@@ -1742,7 +1948,7 @@ static int bnxt_hwrm_ring_free_rx ( struct bnxt *bp )
 
 static int bnxt_hwrm_ring_alloc_nq ( struct bnxt *bp )
 {
-	if ( !bp->thor )
+	if ( ! ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) ) )
 		return STATUS_SUCCESS;
 	return bnxt_hwrm_ring_alloc ( bp, RING_ALLOC_REQ_RING_TYPE_NQ );
 }
@@ -1751,7 +1957,7 @@ static int bnxt_hwrm_ring_free_nq ( struct bnxt *bp )
 {
 	int ret = STATUS_SUCCESS;
 
-	if ( !bp->thor )
+	if ( ! ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) ) )
 		return STATUS_SUCCESS;
 
 	DBGP ( "%s\n", __func__ );
@@ -1822,7 +2028,7 @@ static int bnxt_hwrm_vnic_cfg ( struct bnxt *bp )
 	req->enables = VNIC_CFG_REQ_ENABLES_MRU;
 	req->mru	 = bp->mtu;
 
-	if ( bp->thor ) {
+	if ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) ) {
 		req->enables |= ( VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
 				VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID );
 		req->default_rx_ring_id   = bp->rx_ring_id;
@@ -1876,6 +2082,7 @@ hwrm_func_t bring_up_chip[] = {
 	bnxt_hwrm_backing_store_cfg,	/* HWRM_FUNC_BACKING_STORE_CFG  */
 	bnxt_hwrm_backing_store_qcfg,	/* HWRM_FUNC_BACKING_STORE_QCFG	*/
 	bnxt_hwrm_func_resource_qcaps,	/* HWRM_FUNC_RESOURCE_QCAPS	*/
+	bnxt_hwrm_port_phy_qcaps_req,	/* HWRM_PORT_PHY_QCAPS	*/
 	bnxt_hwrm_func_qcfg_req,	/* HWRM_FUNC_QCFG		*/
 	bnxt_get_vlan,			/* HWRM_NVM_GET_VARIABLE - vlan	*/
 	bnxt_hwrm_port_mac_cfg,		/* HWRM_PORT_MAC_CFG		*/
@@ -1968,6 +2175,9 @@ static int bnxt_tx ( struct net_device *dev, struct io_buffer *iob )
 	bp->tx.iob[entry] = iob;
 	bnxt_set_txq ( bp, entry, mapping, len );
 	entry = NEXT_IDX ( entry, bp->tx.ring_cnt );
+	/* If the ring has wrapped, toggle the epoch bit */
+	if ( bp->tx.prod_id > entry )
+		bp->tx.epoch ^= 1;
 	dump_tx_pkt ( ( u8 * )iob->data, len, bp->tx.prod_id );
 	/* Packets are ready, update Tx producer idx local and on card. */
 	bnxt_db_tx ( bp, ( u32 )entry );
@@ -1986,6 +2196,7 @@ static void bnxt_adv_nq_index ( struct bnxt *bp, u16 cnt )
 	if ( cons_id >= bp->nq.ring_cnt ) {
 		/* Toggle completion bit when the ring wraps. */
 		bp->nq.completion_bit ^= 1;
+		bp->nq.epoch ^= 1;
 		cons_id = cons_id - bp->nq.ring_cnt;
 	}
 	bp->nq.cons_id = cons_id;
@@ -2026,7 +2237,7 @@ static void bnxt_service_cq ( struct net_device *dev )
 
 		cq_type = cmp->type & CMPL_BASE_TYPE_MASK;
 		dump_evt ( ( u8 * )cmp, cq_type, bp->cq.cons_id, 0 );
-		dump_cq ( cmp, bp->cq.cons_id );
+		dump_cq ( cmp, bp->cq.cons_id, bp->nq.toggle );
 
 		switch ( cq_type ) {
 		case CMPL_BASE_TYPE_TX_L2:
@@ -2037,6 +2248,7 @@ static void bnxt_service_cq ( struct net_device *dev )
 			bnxt_adv_cq_index ( bp, 1 );
 			break;
 		case CMPL_BASE_TYPE_RX_L2:
+		case CMPL_BASE_TYPE_RX_L2_V3:
 			done = bnxt_rx_complete ( dev,
 				( struct rx_pkt_cmpl * )cmp );
 			break;
@@ -2063,7 +2275,7 @@ static void bnxt_service_nq ( struct net_device *dev )
 	int done = SERVICE_NEXT_NQ_BD;
 	u32 nq_type;
 
-	if ( !bp->thor )
+	if ( ! ( FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS ) ) )
 		return;
 
 	while ( done == SERVICE_NEXT_NQ_BD ) {
@@ -2072,6 +2284,7 @@ static void bnxt_service_nq ( struct net_device *dev )
 		if ( ( nqp->v & NQ_CN_V ) ^ bp->nq.completion_bit )
 			break;
 		nq_type = ( nqp->type & NQ_CN_TYPE_MASK );
+		bp->nq.toggle = ( ( nqp->type & NQ_CN_TOGGLE_MASK ) >> NQ_CN_TOGGLE_SFT );
 		dump_evt ( ( u8 * )nqp, nq_type, bp->nq.cons_id, 1 );
 		dump_nq ( nqp, bp->nq.cons_id );
 
@@ -2096,8 +2309,8 @@ static void bnxt_service_nq ( struct net_device *dev )
 static void bnxt_poll ( struct net_device *dev )
 {
 	mb (  );
-	bnxt_service_cq ( dev );
 	bnxt_service_nq ( dev );
+	bnxt_service_cq ( dev );
 }
 
 static void bnxt_close ( struct net_device *dev )
diff --git a/src/drivers/net/bnxt/bnxt.h b/src/drivers/net/bnxt/bnxt.h
index 2cbaec5e..8c8a3328 100644
--- a/src/drivers/net/bnxt/bnxt.h
+++ b/src/drivers/net/bnxt/bnxt.h
@@ -52,6 +52,10 @@ union dma_addr64_t {
 #define BNXT_FLAG_NPAR_MODE                     0x0010
 #define BNXT_FLAG_ATOMICS_ENABLE                0x0020
 #define BNXT_FLAG_PCI_VF                        0x0040
+#define BNXT_FLAG_LINK_SPEEDS2                  0x0080
+#define BNXT_FLAG_IS_CHIP_P5                    0x0100
+#define BNXT_FLAG_IS_CHIP_P5_PLUS               0x0200
+#define BNXT_FLAG_IS_CHIP_P7                    0x0400
 /*******************************************************************************
  * Status codes.
  ******************************************************************************/
@@ -106,6 +110,12 @@ union dma_addr64_t {
 #define MEDIUM_SPEED_50GBPS                     0x0a00L
 #define MEDIUM_SPEED_100GBPS                    0x0b00L
 #define MEDIUM_SPEED_200GBPS                    0x0c00L
+#define MEDIUM_SPEED_50PAM4GBPS                 0x0d00L
+#define MEDIUM_SPEED_100PAM4GBPS                0x0e00L
+#define MEDIUM_SPEED_100PAM4_112GBPS            0x0f00L
+#define MEDIUM_SPEED_200PAM4_112GBPS            0x1000L
+#define MEDIUM_SPEED_400PAM4GBPS                0x2000L
+#define MEDIUM_SPEED_400PAM4_112GBPS            0x3000L
 #define MEDIUM_SPEED_AUTONEG_1G_FALLBACK        0x8000L /* Serdes */
 #define MEDIUM_SPEED_AUTONEG_2_5G_FALLBACK      0x8100L /* Serdes */
 #define MEDIUM_SPEED_HARDWARE_DEFAULT           0xff00L /* Serdes nvram def.*/
@@ -168,9 +178,9 @@ union dma_addr64_t {
 	RX_MASK_ACCEPT_MULTICAST)
 #define MAX_NQ_DESC_CNT                         64
 #define NQ_RING_BUFFER_SIZE (MAX_NQ_DESC_CNT * sizeof(struct cmpl_base))
-#define TX_RING_QID (bp->thor ? (u16)bp->queue_id : ((u16)bp->port_idx * 10))
-#define RX_RING_QID (bp->thor ? bp->queue_id : 0)
-#define STAT_CTX_ID ((bp->vf || bp->thor) ? bp->stat_ctx_id : 0)
+#define TX_RING_QID (FLAG_TEST(bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS) ? (u16)bp->queue_id : ((u16)bp->port_idx * 10))
+#define RX_RING_QID (FLAG_TEST(bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS) ? bp->queue_id : 0)
+#define STAT_CTX_ID ((bp->vf || FLAG_TEST(bp->flags, BNXT_FLAG_IS_CHIP_P5_PLUS)) ? bp->stat_ctx_id : 0)
 #define TX_AVAIL(r)                      (r - 1)
 #define TX_IN_USE(a, b, c) ((a - b) & (c - 1))
 #define NO_MORE_NQ_BD_TO_SERVICE         1
@@ -189,13 +199,19 @@ union dma_addr64_t {
 	((idx) << DBC_DBC_INDEX_SFT) & DBC_DBC_INDEX_MASK)
 #define DBC_MSG_XID(xid, flg)  (\
 	(((xid) << DBC_DBC_XID_SFT) & DBC_DBC_XID_MASK) | \
-	DBC_DBC_PATH_L2 | (flg))
+	DBC_DBC_PATH_L2 | (FLAG_TEST ( bp->flags, BNXT_FLAG_IS_CHIP_P7 ) ? DBC_DBC_VALID : 0) | (flg))
+#define DBC_MSG_EPCH(idx)      (\
+        ((idx) << DBC_DBC_EPOCH_SFT))
+#define DBC_MSG_TOGGLE(idx)    (\
+        ((idx) << DBC_DBC_TOGGLE_SFT) & DBC_DBC_TOGGLE_MASK)
 #define PHY_STATUS         0x0001
 #define PHY_SPEED          0x0002
 #define DETECT_MEDIA       0x0004
 #define SUPPORT_SPEEDS     0x0008
+#define SUPPORT_SPEEDS2    0x0010
 #define QCFG_PHY_ALL   (\
-	SUPPORT_SPEEDS | DETECT_MEDIA | PHY_SPEED | PHY_STATUS)
+	SUPPORT_SPEEDS | SUPPORT_SPEEDS2 | \
+        DETECT_MEDIA | PHY_SPEED | PHY_STATUS)
 #define str_mbps           "Mbps"
 #define str_gbps           "Gbps"
 /*
@@ -287,6 +303,18 @@ union dma_addr64_t {
 #define NS_LINK_SPEED_FW_100G                                   (0x6)
 #define LINK_SPEED_FW_200G                                      (0x7L << 7)
 #define NS_LINK_SPEED_FW_200G                                   (0x7)
+#define LINK_SPEED_FW_50G_PAM4                                  (0x8L << 7)
+#define NS_LINK_SPEED_FW_50G_PAM4                               (0x8)
+#define LINK_SPEED_FW_100G_PAM4                                 (0x9L << 7)
+#define NS_LINK_SPEED_FW_100G_PAM4                              (0x9)
+#define LINK_SPEED_FW_100G_PAM4_112                             (0xAL << 7)
+#define NS_LINK_SPEED_FW_100G_PAM4_112                          (0xA)
+#define LINK_SPEED_FW_200G_PAM4_112                             (0xBL << 7)
+#define NS_LINK_SPEED_FW_200G_PAM4_112                          (0xB)
+#define LINK_SPEED_FW_400G_PAM4                                 (0xCL << 7)
+#define NS_LINK_SPEED_FW_400G_PAM4                              (0xC)
+#define LINK_SPEED_FW_400G_PAM4_112                             (0xDL << 7)
+#define NS_LINK_SPEED_FW_400G_PAM4_112                          (0xD)
 #define LINK_SPEED_FW_2_5G                                      (0xEL << 7)
 #define NS_LINK_SPEED_FW_2_5G                                   (0xE)
 #define LINK_SPEED_FW_100M                                      (0xFL << 7)
@@ -387,6 +415,10 @@ struct dbc_dbc {
 	__le32  index;
 	#define DBC_DBC_INDEX_MASK          0xffffffUL
 	#define DBC_DBC_INDEX_SFT           0
+	#define DBC_DBC_EPOCH               0x1000000UL
+	#define DBC_DBC_EPOCH_SFT           24
+	#define DBC_DBC_TOGGLE_MASK         0x6000000UL
+	#define DBC_DBC_TOGGLE_SFT          25
 	__le32  type_path_xid;
 	#define DBC_DBC_XID_MASK            0xfffffUL
 	#define DBC_DBC_XID_SFT             0
@@ -396,6 +428,7 @@ struct dbc_dbc {
 	#define DBC_DBC_PATH_L2             (0x1UL << 24)
 	#define DBC_DBC_PATH_ENGINE         (0x2UL << 24)
 	#define DBC_DBC_PATH_LAST           DBC_DBC_PATH_ENGINE
+	#define DBC_DBC_VALID               0x4000000UL
 	#define DBC_DBC_DEBUG_TRACE         0x8000000UL
 	#define DBC_DBC_TYPE_MASK           0xf0000000UL
 	#define DBC_DBC_TYPE_SFT            28
@@ -481,6 +514,8 @@ struct tx_info {
 	u16              ring_cnt;
 	u32              cnt;   /* Tx statistics. */
 	u32              cnt_req;
+	u8               epoch;
+	u8               res[3];
 };
 
 struct cmpl_base {
@@ -492,6 +527,7 @@ struct cmpl_base {
 #define CMPL_BASE_TYPE_RX_AGG            0x12UL
 #define CMPL_BASE_TYPE_RX_TPA_START      0x13UL
 #define CMPL_BASE_TYPE_RX_TPA_END        0x15UL
+#define CMPL_BASE_TYPE_RX_L2_V3          0x17UL
 #define CMPL_BASE_TYPE_STAT_EJECT        0x1aUL
 #define CMPL_BASE_TYPE_HWRM_DONE         0x20UL
 #define CMPL_BASE_TYPE_HWRM_FWD_REQ      0x22UL
@@ -517,7 +553,8 @@ struct cmp_info {
 	u16       cons_id;
 	u16       ring_cnt;
 	u8        completion_bit;
-	u8        res[3];
+	u8        epoch;
+	u8        res[2];
 };
 
 /* Completion Queue Notification */
@@ -533,6 +570,8 @@ struct nq_base {
  */
 #define NQ_CN_TYPE_MASK           0x3fUL
 #define NQ_CN_TYPE_SFT            0
+#define NQ_CN_TOGGLE_MASK         0xc0UL
+#define NQ_CN_TOGGLE_SFT          6
 /* CQ Notification */
 	    #define NQ_CN_TYPE_CQ_NOTIFICATION  0x30UL
 	    #define NQ_CN_TYPE_LAST            NQ_CN_TYPE_CQ_NOTIFICATION
@@ -561,7 +600,9 @@ struct nq_info {
 	u16       cons_id;
 	u16       ring_cnt;
 	u8        completion_bit;
-	u8        res[3];
+	u8        epoch;
+	u8        toggle;
+	u8        res[1];
 };
 
 struct rx_pkt_cmpl {
@@ -675,6 +716,156 @@ struct rx_pkt_cmpl_hi {
 #define RX_PKT_CMPL_REORDER_SFT  0
 };
 
+struct rx_pkt_v3_cmpl {
+	u16	flags_type;
+	#define RX_PKT_V3_CMPL_TYPE_MASK                      0x3fUL
+	#define RX_PKT_V3_CMPL_TYPE_SFT                       0
+	/*
+	 * RX L2 V3 completion:
+	 * Completion of and L2 RX packet. Length = 32B
+	 * This is the new version of the RX_L2 completion used in Thor2
+	 * and later chips.
+	 */
+	#define RX_PKT_V3_CMPL_TYPE_RX_L2_V3                    0x17UL
+	#define RX_PKT_V3_CMPL_TYPE_LAST                       RX_PKT_V3_CMPL_TYPE_RX_L2_V3
+	#define RX_PKT_V3_CMPL_FLAGS_MASK                     0xffc0UL
+	#define RX_PKT_V3_CMPL_FLAGS_SFT                      6
+	#define RX_PKT_V3_CMPL_FLAGS_ERROR                     0x40UL
+	#define RX_PKT_V3_CMPL_FLAGS_PLACEMENT_MASK            0x380UL
+	#define RX_PKT_V3_CMPL_FLAGS_PLACEMENT_SFT             7
+	#define RX_PKT_V3_CMPL_FLAGS_PLACEMENT_NORMAL            (0x0UL << 7)
+	#define RX_PKT_V3_CMPL_FLAGS_PLACEMENT_JUMBO             (0x1UL << 7)
+	#define RX_PKT_V3_CMPL_FLAGS_PLACEMENT_HDS               (0x2UL << 7)
+	#define RX_PKT_V3_CMPL_FLAGS_PLACEMENT_TRUNCATION        (0x3UL << 7)
+	#define RX_PKT_V3_CMPL_FLAGS_PLACEMENT_LAST             RX_PKT_V3_CMPL_FLAGS_PLACEMENT_TRUNCATION
+	#define RX_PKT_V3_CMPL_FLAGS_RSS_VALID                 0x400UL
+	#define RX_PKT_V3_CMPL_FLAGS_PKT_METADATA_PRESENT      0x800UL
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_MASK                0xf000UL
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_SFT                 12
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_NOT_KNOWN             (0x0UL << 12)
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_IP                    (0x1UL << 12)
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_TCP                   (0x2UL << 12)
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_UDP                   (0x3UL << 12)
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_FCOE                  (0x4UL << 12)
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_ROCE                  (0x5UL << 12)
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_ICMP                  (0x7UL << 12)
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_PTP_WO_TIMESTAMP      (0x8UL << 12)
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_PTP_W_TIMESTAMP       (0x9UL << 12)
+	#define RX_PKT_V3_CMPL_FLAGS_ITYPE_LAST                 RX_PKT_V3_CMPL_FLAGS_ITYPE_PTP_W_TIMESTAMP
+	u16	len;
+	u32	opaque;
+	u16	rss_hash_type_agg_bufs_v1;
+	#define RX_PKT_V3_CMPL_V1                   0x1UL
+	#define RX_PKT_V3_CMPL_AGG_BUFS_MASK        0x3eUL
+	#define RX_PKT_V3_CMPL_AGG_BUFS_SFT         1
+	#define RX_PKT_V3_CMPL_UNUSED1              0x40UL
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_MASK   0xff80UL
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_SFT    7
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_0   (0x0UL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_1   (0x1UL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_3   (0x3UL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_4   (0x4UL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_5   (0x5UL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_6   (0x6UL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_7   (0x7UL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_8   (0x8UL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_9   (0x9UL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_10  (0xaUL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_11  (0xbUL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_12  (0xcUL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_13  (0xdUL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_14  (0xeUL << 7)
+	#define RX_PKT_V3_CMPL_RSS_HASH_TYPE_LAST    RX_PKT_V3_CMPL_RSS_HASH_TYPE_ENUM_14
+	u16	metadata1_payload_offset;
+	#define RX_PKT_V3_CMPL_PAYLOAD_OFFSET_MASK        0x1ffUL
+	#define RX_PKT_V3_CMPL_PAYLOAD_OFFSET_SFT         0
+	#define RX_PKT_V3_CMPL_METADATA1_MASK             0xf000UL
+	#define RX_PKT_V3_CMPL_METADATA1_SFT              12
+	#define RX_PKT_V3_CMPL_METADATA1_TPID_SEL_MASK     0x7000UL
+	#define RX_PKT_V3_CMPL_METADATA1_TPID_SEL_SFT      12
+	#define RX_PKT_V3_CMPL_METADATA1_TPID_SEL_TPID88A8   (0x0UL << 12)
+	#define RX_PKT_V3_CMPL_METADATA1_TPID_SEL_TPID8100   (0x1UL << 12)
+	#define RX_PKT_V3_CMPL_METADATA1_TPID_SEL_TPID9100   (0x2UL << 12)
+	#define RX_PKT_V3_CMPL_METADATA1_TPID_SEL_TPID9200   (0x3UL << 12)
+	#define RX_PKT_V3_CMPL_METADATA1_TPID_SEL_TPID9300   (0x4UL << 12)
+	#define RX_PKT_V3_CMPL_METADATA1_TPID_SEL_TPIDCFG    (0x5UL << 12)
+	#define RX_PKT_V3_CMPL_METADATA1_TPID_SEL_LAST      RX_PKT_V3_CMPL_METADATA1_TPID_SEL_TPIDCFG
+	#define RX_PKT_V3_CMPL_METADATA1_VALID             0x8000UL
+	u32	rss_hash;
+};
+
+struct rx_pkt_v3_cmpl_hi {
+	u32	flags2;
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_IP_CS_CALC                 0x1UL
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_L4_CS_CALC                 0x2UL
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_T_IP_CS_CALC               0x4UL
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_T_L4_CS_CALC               0x8UL
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_META_FORMAT_MASK           0xf0UL
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_META_FORMAT_SFT            4
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_META_FORMAT_NONE             (0x0UL << 4)
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_META_FORMAT_ACT_REC_PTR      (0x1UL << 4)
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_META_FORMAT_TUNNEL_ID        (0x2UL << 4)
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_META_FORMAT_CHDR_DATA        (0x3UL << 4)
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_META_FORMAT_HDR_OFFSET       (0x4UL << 4)
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_META_FORMAT_LAST            RX_PKT_V3_CMPL_HI_FLAGS2_META_FORMAT_HDR_OFFSET
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_IP_TYPE                    0x100UL
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_COMPLETE_CHECKSUM_CALC     0x200UL
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_T_IP_TYPE                  0x400UL
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_T_IP_TYPE_IPV4               (0x0UL << 10)
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_T_IP_TYPE_IPV6               (0x1UL << 10)
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_T_IP_TYPE_LAST              RX_PKT_V3_CMPL_HI_FLAGS2_T_IP_TYPE_IPV6
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_COMPLETE_CHECKSUM_MASK     0xffff0000UL
+	#define RX_PKT_V3_CMPL_HI_FLAGS2_COMPLETE_CHECKSUM_SFT      16
+	u32	metadata2;
+	u16	errors_v2;
+	#define RX_PKT_V3_CMPL_HI_V2                                       0x1UL
+	#define RX_PKT_V3_CMPL_HI_ERRORS_MASK                              0xfffeUL
+	#define RX_PKT_V3_CMPL_HI_ERRORS_SFT                               1
+	#define RX_PKT_V3_CMPL_HI_ERRORS_BUFFER_ERROR_MASK                  0xeUL
+	#define RX_PKT_V3_CMPL_HI_ERRORS_BUFFER_ERROR_SFT                   1
+	#define RX_PKT_V3_CMPL_HI_ERRORS_BUFFER_ERROR_NO_BUFFER               (0x0UL << 1)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_BUFFER_ERROR_DID_NOT_FIT             (0x1UL << 1)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_BUFFER_ERROR_NOT_ON_CHIP             (0x2UL << 1)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_BUFFER_ERROR_BAD_FORMAT              (0x3UL << 1)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_BUFFER_ERROR_FLUSH                   (0x5UL << 1)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_BUFFER_ERROR_LAST                   RX_PKT_V3_CMPL_HI_ERRORS_BUFFER_ERROR_FLUSH
+	#define RX_PKT_V3_CMPL_HI_ERRORS_IP_CS_ERROR                        0x10UL
+	#define RX_PKT_V3_CMPL_HI_ERRORS_L4_CS_ERROR                        0x20UL
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_IP_CS_ERROR                      0x40UL
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_L4_CS_ERROR                      0x80UL
+	#define RX_PKT_V3_CMPL_HI_ERRORS_CRC_ERROR                          0x100UL
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_PKT_ERROR_MASK                   0xe00UL
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_PKT_ERROR_SFT                    9
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_PKT_ERROR_NO_ERROR                 (0x0UL << 9)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_PKT_ERROR_T_L3_BAD_VERSION         (0x1UL << 9)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_PKT_ERROR_T_L3_BAD_HDR_LEN         (0x2UL << 9)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_PKT_ERROR_T_IP_TOTAL_ERROR         (0x3UL << 9)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_PKT_ERROR_T_UDP_TOTAL_ERROR        (0x4UL << 9)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL             (0x5UL << 9)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_PKT_ERROR_T_TOTAL_ERROR            (0x6UL << 9)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_T_PKT_ERROR_LAST                    RX_PKT_V3_CMPL_HI_ERRORS_T_PKT_ERROR_T_TOTAL_ERROR
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_MASK                     0xf000UL
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_SFT                      12
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_NO_ERROR                   (0x0UL << 12)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_L3_BAD_VERSION             (0x1UL << 12)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_L3_BAD_HDR_LEN             (0x2UL << 12)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_L3_BAD_TTL                 (0x3UL << 12)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_IP_TOTAL_ERROR             (0x4UL << 12)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_UDP_TOTAL_ERROR            (0x5UL << 12)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN             (0x6UL << 12)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN_TOO_SMALL   (0x7UL << 12)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN             (0x8UL << 12)
+	#define RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_LAST                      RX_PKT_V3_CMPL_HI_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN
+	u16	metadata0;
+	#define RX_PKT_V3_CMPL_HI_METADATA0_VID_MASK 0xfffUL
+	#define RX_PKT_V3_CMPL_HI_METADATA0_VID_SFT 0
+	#define RX_PKT_V3_CMPL_HI_METADATA0_DE      0x1000UL
+	/* When meta_format=1, this value is the VLAN PRI. */
+	#define RX_PKT_V3_CMPL_HI_METADATA0_PRI_MASK 0xe000UL
+	#define RX_PKT_V3_CMPL_HI_METADATA0_PRI_SFT 13
+	u32	timestamp;
+};
+
 struct rx_prod_pkt_bd {
 	u16  flags_type;
 #define RX_PROD_PKT_BD_TYPE_MASK          0x3fUL
@@ -705,6 +896,8 @@ struct rx_info {
 	u32               drop_err;
 	u32               drop_lb;
 	u32               drop_vlan;
+	u8                epoch;
+	u8                res[3];
 };
 
 #define VALID_DRIVER_REG          0x0001
@@ -750,7 +943,6 @@ struct bnxt {
 	struct nq_info            nq; /* completion info. */
 	u16                       nq_ring_id;
 	u8                        queue_id;
-	u8                        thor;
 	u16                       last_resp_code;
 	u16                       seq_id;
 	u32                       flag_hwrm;
@@ -792,6 +984,7 @@ struct bnxt {
 	u32                       mba_cfg2;
 	u32                       medium;
 	u16                       support_speeds;
+	u16                       auto_link_speeds2_mask;
 	u32                       link_set;
 	u8                        media_detect;
 	u8                        rsvd;
@@ -868,4 +1061,8 @@ struct bnxt {
 	FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR | \
 	FUNC_VF_CFG_REQ_ENABLES_DFLT_MAC_ADDR)
 
-#define CHIP_NUM_57500       0x1750
+#define CHIP_NUM_57508       0x1750
+#define CHIP_NUM_57504       0x1751
+#define CHIP_NUM_57502       0x1752
+
+#define CHIP_NUM_57608       0x1760
diff --git a/src/drivers/net/bnxt/bnxt_dbg.h b/src/drivers/net/bnxt/bnxt_dbg.h
index 188978ad..14540281 100644
--- a/src/drivers/net/bnxt/bnxt_dbg.h
+++ b/src/drivers/net/bnxt/bnxt_dbg.h
@@ -475,7 +475,7 @@ void dbg_rx_stat(struct bnxt *bp)
 #endif
 
 #if defined(DEBUG_CQ)
-static void dump_cq(struct cmpl_base *cmp, u16 cid)
+static void dump_cq(struct cmpl_base *cmp, u16 cid, u8 toggle)
 {
 	dbg_prn("- CQ Type ");
 	switch (cmp->type & CMPL_BASE_TYPE_MASK) {
@@ -495,7 +495,7 @@ static void dump_cq(struct cmpl_base *cmp, u16 cid)
 		dbg_prn("%04x", (u16)(cmp->type & CMPL_BASE_TYPE_MASK));
 		break;
 	}
-	dbg_prn(" cid %d", cid);
+	dbg_prn(" cid %d, tog %d", cid, toggle);
 #if defined(DEBUG_CQ_DUMP)
 	dump_mem((u8 *)cmp, (u32)sizeof(struct cmpl_base), DISP_U8);
 #else
@@ -513,7 +513,7 @@ static void dump_nq(struct nq_base *nqp, u16 cid)
 #endif
 }
 #else
-#define dump_cq(cq, id)
+#define dump_cq(cq, id, toggle)
 #define dump_nq(nq, id)
 #endif
 
diff --git a/src/drivers/net/bnxt/bnxt_hsi.h b/src/drivers/net/bnxt/bnxt_hsi.h
index 086acb8b..dbcffd90 100644
--- a/src/drivers/net/bnxt/bnxt_hsi.h
+++ b/src/drivers/net/bnxt/bnxt_hsi.h
@@ -2929,7 +2929,7 @@ struct hwrm_func_drv_if_change_output {
 	u8	valid;
 };
 
-/* hwrm_port_phy_cfg_input (size:448b/56B) */
+/* hwrm_port_phy_cfg_input (size:512b/64B) */
 struct hwrm_port_phy_cfg_input {
 	__le16	req_type;
 	__le16	cmpl_ring;
@@ -2952,6 +2952,15 @@ struct hwrm_port_phy_cfg_input {
 	#define PORT_PHY_CFG_REQ_FLAGS_FEC_CLAUSE91_ENABLE      0x1000UL
 	#define PORT_PHY_CFG_REQ_FLAGS_FEC_CLAUSE91_DISABLE     0x2000UL
 	#define PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN           0x4000UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS544_1XN_ENABLE     0x8000UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS544_1XN_DISABLE    0x10000UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS544_IEEE_ENABLE    0x20000UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS544_IEEE_DISABLE   0x40000UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS272_1XN_ENABLE     0x80000UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS272_1XN_DISABLE    0x100000UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS272_IEEE_ENABLE    0x200000UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_RS272_IEEE_DISABLE   0x400000UL
+
 	__le32	enables;
 	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE                0x1UL
 	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_DUPLEX              0x2UL
@@ -2964,6 +2973,10 @@ struct hwrm_port_phy_cfg_input {
 	#define PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE              0x100UL
 	#define PORT_PHY_CFG_REQ_ENABLES_EEE_LINK_SPEED_MASK      0x200UL
 	#define PORT_PHY_CFG_REQ_ENABLES_TX_LPI_TIMER             0x400UL
+	#define PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED         0x800UL
+	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK     0x1000UL
+	#define PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2            0x2000UL
+	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEEDS2_MASK        0x4000UL
 	__le16	port_id;
 	__le16	force_link_speed;
 	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_100MB 0x1UL
@@ -3049,11 +3062,48 @@ struct hwrm_port_phy_cfg_input {
 	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_RSVD3     0x10UL
 	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_RSVD4     0x20UL
 	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_10GB      0x40UL
-	u8	unused_2[2];
+	__le16	force_pam4_link_speed;
+	#define PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_50GB  0x1f4UL
+	#define PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_100GB 0x3e8UL
+	#define PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_200GB 0x7d0UL
+	#define PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_LAST PORT_PHY_CFG_REQ_FORCE_PAM4_LINK_SPEED_200GB
 	__le32	tx_lpi_timer;
 	#define PORT_PHY_CFG_REQ_TX_LPI_TIMER_MASK 0xffffffUL
 	#define PORT_PHY_CFG_REQ_TX_LPI_TIMER_SFT 0
-	__le32	unused_3;
+	__le16	auto_link_pam4_speed_mask;
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_PAM4_SPEED_MASK_50G      0x1UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_PAM4_SPEED_MASK_100G     0x2UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_PAM4_SPEED_MASK_200G     0x4UL
+	__le16	force_link_speeds2;
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_1GB            0xaUL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_10GB           0x64UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_25GB           0xfaUL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_40GB           0x190UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_50GB           0x1f4UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_100GB          0x3e8UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_50GB_PAM4_56   0x1f5UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_100GB_PAM4_56  0x3e9UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_200GB_PAM4_56  0x7d1UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_400GB_PAM4_56  0xfa1UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_100GB_PAM4_112 0x3eaUL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_200GB_PAM4_112 0x7d2UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_400GB_PAM4_112 0xfa2UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_LAST          PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_400GB_PAM4_112
+	__le16	auto_link_speeds2_mask;
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_1GB                0x1UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_10GB               0x2UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_25GB               0x4UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_40GB               0x8UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_50GB               0x10UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_100GB              0x20UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_50GB_PAM4_56       0x40UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_100GB_PAM4_56      0x80UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_200GB_PAM4_56      0x100UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_400GB_PAM4_56      0x200UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_100GB_PAM4_112     0x400UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_200GB_PAM4_112     0x800UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEEDS2_MASK_400GB_PAM4_112     0x1000UL
+	u8	unused_2[6];
 };
 
 /* hwrm_port_phy_cfg_output (size:128b/16B) */
@@ -3087,7 +3137,7 @@ struct hwrm_port_phy_qcfg_input {
 	u8	unused_0[6];
 };
 
-/* hwrm_port_phy_qcfg_output (size:768b/96B) */
+/* hwrm_port_phy_qcfg_output (size:832b/104B) */
 struct hwrm_port_phy_qcfg_output {
 	__le16	error_code;
 	__le16	req_type;
@@ -3098,7 +3148,23 @@ struct hwrm_port_phy_qcfg_output {
 	#define PORT_PHY_QCFG_RESP_LINK_SIGNAL  0x1UL
 	#define PORT_PHY_QCFG_RESP_LINK_LINK    0x2UL
 	#define PORT_PHY_QCFG_RESP_LINK_LAST   PORT_PHY_QCFG_RESP_LINK_LINK
-	u8	unused_0;
+	u8	active_fec_signal_mode;
+	#define PORT_PHY_QCFG_RESP_SIGNAL_MODE_MASK     0xfUL
+	#define PORT_PHY_QCFG_RESP_SIGNAL_MODE_SFT      0
+	#define PORT_PHY_QCFG_RESP_SIGNAL_MODE_NRZ      0x0UL
+	#define PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4     0x1UL
+	#define PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4_112 0x2UL
+	#define PORT_PHY_QCFG_RESP_SIGNAL_MODE_LAST     HWRM_PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4_112
+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK                   0xf0UL
+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_SFT                    4
+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE        (0x0UL << 4)
+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE    (0x1UL << 4)
+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE    (0x2UL << 4)
+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE   (0x3UL << 4)
+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE  (0x4UL << 4)
+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE   (0x5UL << 4)
+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE  (0x6UL << 4)
+	#define PORT_PHY_QCFG_RESP_ACTIVE_FEC_LAST                   PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE
 	__le16	link_speed;
 	#define PORT_PHY_QCFG_RESP_LINK_SPEED_100MB 0x1UL
 	#define PORT_PHY_QCFG_RESP_LINK_SPEED_1GB   0xaUL
@@ -3111,6 +3177,7 @@ struct hwrm_port_phy_qcfg_output {
 	#define PORT_PHY_QCFG_RESP_LINK_SPEED_50GB  0x1f4UL
 	#define PORT_PHY_QCFG_RESP_LINK_SPEED_100GB 0x3e8UL
 	#define PORT_PHY_QCFG_RESP_LINK_SPEED_200GB 0x7d0UL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_400GB 0xfa0UL
 	#define PORT_PHY_QCFG_RESP_LINK_SPEED_10MB  0xffffUL
 	#define PORT_PHY_QCFG_RESP_LINK_SPEED_LAST PORT_PHY_QCFG_RESP_LINK_SPEED_10MB
 	u8	duplex_cfg;
@@ -3249,7 +3316,31 @@ struct hwrm_port_phy_qcfg_output {
 	#define PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASESR4     0x1dUL
 	#define PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASELR4     0x1eUL
 	#define PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASEER4     0x1fUL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_LAST            PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASEER4
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_50G_BASECR       0x20UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_50G_BASESR       0x21UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_50G_BASELR       0x22UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_50G_BASEER       0x23UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASECR2     0x24UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASESR2     0x25UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASELR2     0x26UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASEER2     0x27UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASECR      0x28UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASESR      0x29UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASELR      0x2aUL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASEER      0x2bUL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASECR2     0x2cUL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASESR2     0x2dUL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASELR2     0x2eUL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASEER2     0x2fUL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASECR8     0x30UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASESR8     0x31UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASELR8     0x32UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASEER8     0x33UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASECR4     0x34UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASESR4     0x35UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASELR4     0x36UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASEER4     0x37UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_LAST            PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASEER4
 	u8	media_type;
 	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_UNKNOWN 0x0UL
 	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_TP      0x1UL
@@ -3330,15 +3421,90 @@ struct hwrm_port_phy_qcfg_output {
 	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE74_ENABLED       0x10UL
 	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE91_SUPPORTED     0x20UL
 	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE91_ENABLED       0x40UL
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS544_1XN_SUPPORTED    0x80UL
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS544_1XN_ENABLED      0x100UL
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS544_IEEE_SUPPORTED   0x200UL
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS544_IEEE_ENABLED     0x400UL
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS272_1XN_SUPPORTED    0x800UL
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS272_1XN_ENABLED      0x1000UL
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS272_IEEE_SUPPORTED   0x2000UL
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS272_IEEE_ENABLED     0x4000UL
 	u8	duplex_state;
 	#define PORT_PHY_QCFG_RESP_DUPLEX_STATE_HALF 0x0UL
 	#define PORT_PHY_QCFG_RESP_DUPLEX_STATE_FULL 0x1UL
 	#define PORT_PHY_QCFG_RESP_DUPLEX_STATE_LAST PORT_PHY_QCFG_RESP_DUPLEX_STATE_FULL
 	u8	option_flags;
 	#define PORT_PHY_QCFG_RESP_OPTION_FLAGS_MEDIA_AUTO_DETECT     0x1UL
+	#define PORT_PHY_QCFG_RESP_OPTION_FLAGS_SIGNAL_MODE_KNOWN     0x2UL
+	#define PORT_PHY_QCFG_RESP_OPTION_FLAGS_SPEEDS2_SUPPORTED     0x4UL
 	char	phy_vendor_name[16];
 	char	phy_vendor_partnumber[16];
-	u8	unused_2[7];
+	__le16	support_pam4_speeds;
+	#define PORT_PHY_QCFG_RESP_SUPPORT_PAM4_SPEEDS_50G      0x1UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_PAM4_SPEEDS_100G     0x2UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_PAM4_SPEEDS_200G     0x4UL
+	__le16	force_pam4_link_speed;
+	#define PORT_PHY_QCFG_RESP_FORCE_PAM4_LINK_SPEED_50GB  0x1f4UL
+	#define PORT_PHY_QCFG_RESP_FORCE_PAM4_LINK_SPEED_100GB 0x3e8UL
+	#define PORT_PHY_QCFG_RESP_FORCE_PAM4_LINK_SPEED_200GB 0x7d0UL
+	#define PORT_PHY_QCFG_RESP_FORCE_PAM4_LINK_SPEED_LAST PORT_PHY_QCFG_RESP_FORCE_PAM4_LINK_SPEED_200GB
+	__le16	auto_pam4_link_speed_mask;
+	#define PORT_PHY_QCFG_RESP_AUTO_PAM4_LINK_SPEED_MASK_50G      0x1UL
+	#define PORT_PHY_QCFG_RESP_AUTO_PAM4_LINK_SPEED_MASK_100G     0x2UL
+	#define PORT_PHY_QCFG_RESP_AUTO_PAM4_LINK_SPEED_MASK_200G     0x4UL
+	u8	link_partner_pam4_adv_speeds;
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_PAM4_ADV_SPEEDS_50GB      0x1UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_PAM4_ADV_SPEEDS_100GB     0x2UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_PAM4_ADV_SPEEDS_200GB     0x4UL
+	u8	link_down_reason;
+	#define PORT_PHY_QCFG_RESP_LINK_DOWN_REASON_RF     0x1UL
+	__le16	support_speeds2;
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_1GB                0x1UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_10GB               0x2UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_25GB               0x4UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_40GB               0x8UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_50GB               0x10UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_100GB              0x20UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_50GB_PAM4_56       0x40UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_100GB_PAM4_56      0x80UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_200GB_PAM4_56      0x100UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_400GB_PAM4_56      0x200UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_100GB_PAM4_112     0x400UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_200GB_PAM4_112     0x800UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_400GB_PAM4_112     0x1000UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_800GB_PAM4_112     0x2000UL
+	__le16	force_link_speeds2;
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_1GB            0xaUL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_10GB           0x64UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_25GB           0xfaUL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_40GB           0x190UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_50GB           0x1f4UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_100GB          0x3e8UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_50GB_PAM4_56   0x1f5UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_100GB_PAM4_56  0x3e9UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_200GB_PAM4_56  0x7d1UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_400GB_PAM4_56  0xfa1UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_100GB_PAM4_112 0x3eaUL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_200GB_PAM4_112 0x7d2UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_400GB_PAM4_112 0xfa2UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_800GB_PAM4_112 0x1f42UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_LAST          PORT_PHY_QCFG_RESP_FORCE_LINK_SPEEDS2_800GB_PAM4_112
+	__le16	auto_link_speeds2;
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_1GB                0x1UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_10GB               0x2UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_25GB               0x4UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_40GB               0x8UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_50GB               0x10UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_100GB              0x20UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_50GB_PAM4_56       0x40UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_100GB_PAM4_56      0x80UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_200GB_PAM4_56      0x100UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_400GB_PAM4_56      0x200UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_100GB_PAM4_112     0x400UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_200GB_PAM4_112     0x800UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_400GB_PAM4_112     0x1000UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEEDS2_800GB_PAM4_112     0x2000UL
+	u8	active_lanes;
 	u8	valid;
 };
 
@@ -3888,7 +4054,7 @@ struct hwrm_port_phy_qcaps_input {
 	u8	unused_0[6];
 };
 
-/* hwrm_port_phy_qcaps_output (size:192b/24B) */
+/* hwrm_port_phy_qcaps_output (size:320b/40B) */
 struct hwrm_port_phy_qcaps_output {
 	__le16	error_code;
 	__le16	req_type;
@@ -3954,6 +4120,53 @@ struct hwrm_port_phy_qcaps_output {
 	#define PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_SFT 0
 	#define PORT_PHY_QCAPS_RESP_VALID_MASK            0xff000000UL
 	#define PORT_PHY_QCAPS_RESP_VALID_SFT             24
+	__le16	supported_pam4_speeds_auto_mode;
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_PAM4_SPEEDS_AUTO_MODE_50G      0x1UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_PAM4_SPEEDS_AUTO_MODE_100G     0x2UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_PAM4_SPEEDS_AUTO_MODE_200G     0x4UL
+	__le16	supported_pam4_speeds_force_mode;
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_PAM4_SPEEDS_FORCE_MODE_50G      0x1UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_PAM4_SPEEDS_FORCE_MODE_100G     0x2UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_PAM4_SPEEDS_FORCE_MODE_200G     0x4UL
+	__le16	flags2;
+	#define PORT_PHY_QCAPS_RESP_FLAGS2_PAUSE_UNSUPPORTED       0x1UL
+	#define PORT_PHY_QCAPS_RESP_FLAGS2_PFC_UNSUPPORTED         0x2UL
+	#define PORT_PHY_QCAPS_RESP_FLAGS2_BANK_ADDR_SUPPORTED     0x4UL
+	#define PORT_PHY_QCAPS_RESP_FLAGS2_SPEEDS2_SUPPORTED       0x8UL
+	u8	internal_port_cnt;
+	u8	unused_0;
+	__le16	supported_speeds2_force_mode;
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_1GB                0x1UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_10GB               0x2UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_25GB               0x4UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_40GB               0x8UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_50GB               0x10UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_100GB              0x20UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_50GB_PAM4_56       0x40UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_100GB_PAM4_56      0x80UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_200GB_PAM4_56      0x100UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_400GB_PAM4_56      0x200UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_100GB_PAM4_112     0x400UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_200GB_PAM4_112     0x800UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_400GB_PAM4_112     0x1000UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_FORCE_MODE_800GB_PAM4_112     0x2000UL
+	__le16	supported_speeds2_auto_mode;
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_1GB                0x1UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_10GB               0x2UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_25GB               0x4UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_40GB               0x8UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_50GB               0x10UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_100GB              0x20UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_50GB_PAM4_56       0x40UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_100GB_PAM4_56      0x80UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_200GB_PAM4_56      0x100UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_400GB_PAM4_56      0x200UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_100GB_PAM4_112     0x400UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_200GB_PAM4_112     0x800UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_400GB_PAM4_112     0x1000UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS2_AUTO_MODE_800GB_PAM4_112     0x2000UL
+	u8	unused_1[3];
+	u8	valid;
 };
 
 /* hwrm_port_phy_i2c_write_input (size:832b/104B) */
diff --git a/src/drivers/net/davicom.c b/src/drivers/net/davicom.c
index 9d3d8b91..0c96796d 100644
--- a/src/drivers/net/davicom.c
+++ b/src/drivers/net/davicom.c
@@ -689,9 +689,9 @@ static struct nic_operations davicom_operations = {
 };
 
 static struct pci_device_id davicom_nics[] = {
+PCI_ROM(0x1282, 0x9009, "davicom9009", "Davicom 9009", 0),
 PCI_ROM(0x1282, 0x9100, "davicom9100", "Davicom 9100", 0),
 PCI_ROM(0x1282, 0x9102, "davicom9102", "Davicom 9102", 0),
-PCI_ROM(0x1282, 0x9009, "davicom9009", "Davicom 9009", 0),
 PCI_ROM(0x1282, 0x9132, "davicom9132", "Davicom 9132", 0),	/* Needs probably some fixing */
 };
 
diff --git a/src/drivers/net/dmfe.c b/src/drivers/net/dmfe.c
index 2ea0d2b2..53b05815 100644
--- a/src/drivers/net/dmfe.c
+++ b/src/drivers/net/dmfe.c
@@ -1208,9 +1208,9 @@ static struct nic_operations dmfe_operations = {
 };
 
 static struct pci_device_id dmfe_nics[] = {
+	PCI_ROM(0x1282, 0x9009, "dmfe9009", "Davicom 9009", 0),
 	PCI_ROM(0x1282, 0x9100, "dmfe9100", "Davicom 9100", 0),
 	PCI_ROM(0x1282, 0x9102, "dmfe9102", "Davicom 9102", 0),
-	PCI_ROM(0x1282, 0x9009, "dmfe9009", "Davicom 9009", 0),
 	PCI_ROM(0x1282, 0x9132, "dmfe9132", "Davicom 9132", 0),	/* Needs probably some fixing */
 };
 
diff --git a/src/drivers/net/eepro100.c b/src/drivers/net/eepro100.c
index a0551a89..49b00d44 100644
--- a/src/drivers/net/eepro100.c
+++ b/src/drivers/net/eepro100.c
@@ -1126,8 +1126,12 @@ PCI_ROM(0x8086, 0x103b, "82562etb",      "Intel PRO100 VE 82562ETB", 0),
 PCI_ROM(0x8086, 0x103c, "eepro100-103c", "Intel PRO/100 VM Network Connection", 0),
 PCI_ROM(0x8086, 0x103d, "eepro100-103d", "Intel PRO/100 VE Network Connection", 0),
 PCI_ROM(0x8086, 0x103e, "eepro100-103e", "Intel PRO/100 VM Network Connection", 0),
+PCI_ROM(0x8086, 0x1050, "82562ez",       "Intel 82562EZ Network Connection", 0),
 PCI_ROM(0x8086, 0x1051, "prove",         "Intel PRO/100 VE Network Connection", 0),
 PCI_ROM(0x8086, 0x1059, "82551qm",       "Intel PRO/100 M Mobile Connection", 0),
+PCI_ROM(0x8086, 0x1065, "82562-3",       "Intel 82562 based Fast Ethernet Connection", 0),
+PCI_ROM(0x8086, 0x1092, "82562-3",       "Intel Pro/100 VE Network", 0),
+PCI_ROM(0x8086, 0x10fe, "82552",         "Intel 82552 10/100 Network Connection", 0),
 PCI_ROM(0x8086, 0x1209, "82559er",       "Intel EtherExpressPro100 82559ER", 0),
 PCI_ROM(0x8086, 0x1227, "82865",         "Intel 82865 EtherExpress PRO/100A", 0),
 PCI_ROM(0x8086, 0x1228, "82556",         "Intel 82556 EtherExpress PRO/100 Smart", 0),
@@ -1135,13 +1139,9 @@ PCI_ROM(0x8086, 0x1229, "eepro100",      "Intel EtherExpressPro100", 0),
 PCI_ROM(0x8086, 0x2449, "82562em",       "Intel EtherExpressPro100 82562EM", 0),
 PCI_ROM(0x8086, 0x2459, "82562-1",       "Intel 82562 based Fast Ethernet Connection", 0),
 PCI_ROM(0x8086, 0x245d, "82562-2",       "Intel 82562 based Fast Ethernet Connection", 0),
-PCI_ROM(0x8086, 0x1050, "82562ez",       "Intel 82562EZ Network Connection", 0),
-PCI_ROM(0x8086, 0x1065, "82562-3",       "Intel 82562 based Fast Ethernet Connection", 0),
+PCI_ROM(0x8086, 0x27dc, "eepro100-27dc", "Intel 82801G (ICH7) Chipset Ethernet Controller", 0),
 PCI_ROM(0x8086, 0x5200, "eepro100-5200", "Intel EtherExpress PRO/100 Intelligent Server", 0),
 PCI_ROM(0x8086, 0x5201, "eepro100-5201", "Intel EtherExpress PRO/100 Intelligent Server", 0),
-PCI_ROM(0x8086, 0x1092, "82562-3",       "Intel Pro/100 VE Network", 0),
-PCI_ROM(0x8086, 0x27dc, "eepro100-27dc", "Intel 82801G (ICH7) Chipset Ethernet Controller", 0),
-PCI_ROM(0x8086, 0x10fe, "82552",         "Intel 82552 10/100 Network Connection", 0),
 };
 
 /* Cards with device ids 0x1030 to 0x103F, 0x2449, 0x2459 or 0x245D might need
diff --git a/src/drivers/net/forcedeth.c b/src/drivers/net/forcedeth.c
index ec3a5bdb..b4019d04 100644
--- a/src/drivers/net/forcedeth.c
+++ b/src/drivers/net/forcedeth.c
@@ -1928,17 +1928,17 @@ forcedeth_remove ( struct pci_device *pdev )
 }
 
 static struct pci_device_id forcedeth_nics[] = {
-	PCI_ROM(0x10DE, 0x01C3, "nForce", "nForce Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER),
+	PCI_ROM(0x10DE, 0x0037, "MCP04", "MCP04 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC|DEV_HAS_CHECKSUM| DEV_HAS_HIGH_DMA|DEV_HAS_STATISTICS_V1|DEV_NEED_TX_LIMIT),
+	PCI_ROM(0x10DE, 0x0038, "MCP04", "MCP04 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC|DEV_HAS_CHECKSUM| DEV_HAS_HIGH_DMA|DEV_HAS_STATISTICS_V1|DEV_NEED_TX_LIMIT),
+	PCI_ROM(0x10DE, 0x0056, "CK804", "CK804 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC|DEV_HAS_CHECKSUM| DEV_HAS_HIGH_DMA|DEV_HAS_STATISTICS_V1|DEV_NEED_TX_LIMIT),
+	PCI_ROM(0x10DE, 0x0057, "CK804", "CK804 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC|DEV_HAS_CHECKSUM| DEV_HAS_HIGH_DMA|DEV_HAS_STATISTICS_V1|DEV_NEED_TX_LIMIT),
 	PCI_ROM(0x10DE, 0x0066, "nForce2", "nForce2 Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER),
-	PCI_ROM(0x10DE, 0x00D6, "nForce3", "nForce3 Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER),
 	PCI_ROM(0x10DE, 0x0086, "nForce3", "nForce3 Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC| DEV_HAS_CHECKSUM),
 	PCI_ROM(0x10DE, 0x008C, "nForce3", "nForce3 Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC| DEV_HAS_CHECKSUM),
-	PCI_ROM(0x10DE, 0x00E6, "nForce3", "nForce3 Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC| DEV_HAS_CHECKSUM),
+	PCI_ROM(0x10DE, 0x00D6, "nForce3", "nForce3 Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER),
 	PCI_ROM(0x10DE, 0x00DF, "nForce3", "nForce3 Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC| DEV_HAS_CHECKSUM),
-	PCI_ROM(0x10DE, 0x0056, "CK804", "CK804 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC|DEV_HAS_CHECKSUM| DEV_HAS_HIGH_DMA|DEV_HAS_STATISTICS_V1|DEV_NEED_TX_LIMIT),
-	PCI_ROM(0x10DE, 0x0057, "CK804", "CK804 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC|DEV_HAS_CHECKSUM| DEV_HAS_HIGH_DMA|DEV_HAS_STATISTICS_V1|DEV_NEED_TX_LIMIT),
-	PCI_ROM(0x10DE, 0x0037, "MCP04", "MCP04 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC|DEV_HAS_CHECKSUM| DEV_HAS_HIGH_DMA|DEV_HAS_STATISTICS_V1|DEV_NEED_TX_LIMIT),
-	PCI_ROM(0x10DE, 0x0038, "MCP04", "MCP04 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC|DEV_HAS_CHECKSUM| DEV_HAS_HIGH_DMA|DEV_HAS_STATISTICS_V1|DEV_NEED_TX_LIMIT),
+	PCI_ROM(0x10DE, 0x00E6, "nForce3", "nForce3 Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC| DEV_HAS_CHECKSUM),
+	PCI_ROM(0x10DE, 0x01C3, "nForce", "nForce Ethernet Controller", DEV_NEED_TIMERIRQ|DEV_NEED_LINKTIMER),
 	PCI_ROM(0x10DE, 0x0268, "MCP51", "MCP51 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_STATISTICS_V1|DEV_NEED_LOW_POWER_FIX),
 	PCI_ROM(0x10DE, 0x0269, "MCP51", "MCP51 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_STATISTICS_V1|DEV_NEED_LOW_POWER_FIX),
 	PCI_ROM(0x10DE, 0x0372, "MCP55", "MCP55 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC|DEV_HAS_CHECKSUM| DEV_HAS_HIGH_DMA|DEV_HAS_VLAN|DEV_HAS_MSI|DEV_HAS_MSI_X| DEV_HAS_POWER_CNTRL|DEV_HAS_PAUSEFRAME_TX_V1| DEV_HAS_STATISTICS_V2|DEV_HAS_TEST_EXTENDED| DEV_HAS_MGMT_UNIT|DEV_NEED_TX_LIMIT|DEV_NEED_MSI_FIX),
@@ -1955,14 +1955,14 @@ static struct pci_device_id forcedeth_nics[] = {
 	PCI_ROM(0x10DE, 0x054D, "MCP67", "MCP67 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_MSI|DEV_HAS_PAUSEFRAME_TX_V1|DEV_HAS_STATISTICS_V2| DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT|DEV_HAS_CORRECT_MACADDR| DEV_HAS_GEAR_MODE|DEV_NEED_MSI_FIX),
 	PCI_ROM(0x10DE, 0x054E, "MCP67", "MCP67 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_MSI|DEV_HAS_PAUSEFRAME_TX_V1|DEV_HAS_STATISTICS_V2| DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT|DEV_HAS_CORRECT_MACADDR| DEV_HAS_GEAR_MODE|DEV_NEED_MSI_FIX),
 	PCI_ROM(0x10DE, 0x054F, "MCP67", "MCP67 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_MSI|DEV_HAS_PAUSEFRAME_TX_V1|DEV_HAS_STATISTICS_V2| DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT|DEV_HAS_CORRECT_MACADDR| DEV_HAS_GEAR_MODE|DEV_NEED_MSI_FIX),
-	PCI_ROM(0x10DE, 0x07DC, "MCP73", "MCP73 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_MSI|DEV_HAS_PAUSEFRAME_TX_V1|DEV_HAS_STATISTICS_V2| DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT|DEV_HAS_CORRECT_MACADDR| DEV_HAS_COLLISION_FIX|DEV_HAS_GEAR_MODE|DEV_NEED_MSI_FIX),
-	PCI_ROM(0x10DE, 0x07DD, "MCP73", "MCP73 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_MSI|DEV_HAS_PAUSEFRAME_TX_V1|DEV_HAS_STATISTICS_V2| DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT|DEV_HAS_CORRECT_MACADDR| DEV_HAS_COLLISION_FIX|DEV_HAS_GEAR_MODE|DEV_NEED_MSI_FIX),
-	PCI_ROM(0x10DE, 0x07DE, "MCP73", "MCP73 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_MSI|DEV_HAS_PAUSEFRAME_TX_V1|DEV_HAS_STATISTICS_V2| DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT|DEV_HAS_CORRECT_MACADDR| DEV_HAS_COLLISION_FIX|DEV_HAS_GEAR_MODE|DEV_NEED_MSI_FIX),
-	PCI_ROM(0x10DE, 0x07DF, "MCP73", "MCP73 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_MSI|DEV_HAS_PAUSEFRAME_TX_V1|DEV_HAS_STATISTICS_V2| DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT|DEV_HAS_CORRECT_MACADDR| DEV_HAS_COLLISION_FIX|DEV_HAS_GEAR_MODE|DEV_NEED_MSI_FIX),
 	PCI_ROM(0x10DE, 0x0760, "MCP77", "MCP77 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_CHECKSUM|DEV_HAS_HIGH_DMA| DEV_HAS_MSI|DEV_HAS_POWER_CNTRL|DEV_HAS_PAUSEFRAME_TX_V2| DEV_HAS_STATISTICS_V3|DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT| DEV_HAS_CORRECT_MACADDR|DEV_HAS_COLLISION_FIX| DEV_NEED_TX_LIMIT2|DEV_HAS_GEAR_MODE|DEV_NEED_PHY_INIT_FIX| DEV_NEED_MSI_FIX),
 	PCI_ROM(0x10DE, 0x0761, "MCP77", "MCP77 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_CHECKSUM|DEV_HAS_HIGH_DMA| DEV_HAS_MSI|DEV_HAS_POWER_CNTRL|DEV_HAS_PAUSEFRAME_TX_V2| DEV_HAS_STATISTICS_V3|DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT| DEV_HAS_CORRECT_MACADDR|DEV_HAS_COLLISION_FIX| DEV_NEED_TX_LIMIT2|DEV_HAS_GEAR_MODE|DEV_NEED_PHY_INIT_FIX| DEV_NEED_MSI_FIX),
 	PCI_ROM(0x10DE, 0x0762, "MCP77", "MCP77 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_CHECKSUM|DEV_HAS_HIGH_DMA| DEV_HAS_MSI|DEV_HAS_POWER_CNTRL|DEV_HAS_PAUSEFRAME_TX_V2| DEV_HAS_STATISTICS_V3|DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT| DEV_HAS_CORRECT_MACADDR|DEV_HAS_COLLISION_FIX| DEV_NEED_TX_LIMIT2|DEV_HAS_GEAR_MODE|DEV_NEED_PHY_INIT_FIX| DEV_NEED_MSI_FIX),
 	PCI_ROM(0x10DE, 0x0763, "MCP77", "MCP77 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_CHECKSUM|DEV_HAS_HIGH_DMA| DEV_HAS_MSI|DEV_HAS_POWER_CNTRL|DEV_HAS_PAUSEFRAME_TX_V2| DEV_HAS_STATISTICS_V3|DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT| DEV_HAS_CORRECT_MACADDR|DEV_HAS_COLLISION_FIX| DEV_NEED_TX_LIMIT2|DEV_HAS_GEAR_MODE|DEV_NEED_PHY_INIT_FIX| DEV_NEED_MSI_FIX),
+	PCI_ROM(0x10DE, 0x07DC, "MCP73", "MCP73 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_MSI|DEV_HAS_PAUSEFRAME_TX_V1|DEV_HAS_STATISTICS_V2| DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT|DEV_HAS_CORRECT_MACADDR| DEV_HAS_COLLISION_FIX|DEV_HAS_GEAR_MODE|DEV_NEED_MSI_FIX),
+	PCI_ROM(0x10DE, 0x07DD, "MCP73", "MCP73 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_MSI|DEV_HAS_PAUSEFRAME_TX_V1|DEV_HAS_STATISTICS_V2| DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT|DEV_HAS_CORRECT_MACADDR| DEV_HAS_COLLISION_FIX|DEV_HAS_GEAR_MODE|DEV_NEED_MSI_FIX),
+	PCI_ROM(0x10DE, 0x07DE, "MCP73", "MCP73 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_MSI|DEV_HAS_PAUSEFRAME_TX_V1|DEV_HAS_STATISTICS_V2| DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT|DEV_HAS_CORRECT_MACADDR| DEV_HAS_COLLISION_FIX|DEV_HAS_GEAR_MODE|DEV_NEED_MSI_FIX),
+	PCI_ROM(0x10DE, 0x07DF, "MCP73", "MCP73 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_HIGH_DMA|DEV_HAS_POWER_CNTRL| DEV_HAS_MSI|DEV_HAS_PAUSEFRAME_TX_V1|DEV_HAS_STATISTICS_V2| DEV_HAS_TEST_EXTENDED|DEV_HAS_MGMT_UNIT|DEV_HAS_CORRECT_MACADDR| DEV_HAS_COLLISION_FIX|DEV_HAS_GEAR_MODE|DEV_NEED_MSI_FIX),
 	PCI_ROM(0x10DE, 0x0AB0, "MCP79", "MCP79 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC|DEV_HAS_CHECKSUM| DEV_HAS_HIGH_DMA|DEV_HAS_MSI|DEV_HAS_POWER_CNTRL| DEV_HAS_PAUSEFRAME_TX_V3|DEV_HAS_STATISTICS_V3| DEV_HAS_TEST_EXTENDED|DEV_HAS_CORRECT_MACADDR| DEV_HAS_COLLISION_FIX|DEV_NEED_TX_LIMIT2|DEV_HAS_GEAR_MODE| DEV_NEED_PHY_INIT_FIX|DEV_NEED_MSI_FIX),
 	PCI_ROM(0x10DE, 0x0AB1, "MCP79", "MCP79 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC|DEV_HAS_CHECKSUM| DEV_HAS_HIGH_DMA|DEV_HAS_MSI|DEV_HAS_POWER_CNTRL| DEV_HAS_PAUSEFRAME_TX_V3|DEV_HAS_STATISTICS_V3| DEV_HAS_TEST_EXTENDED|DEV_HAS_CORRECT_MACADDR| DEV_HAS_COLLISION_FIX|DEV_NEED_TX_LIMIT2|DEV_HAS_GEAR_MODE| DEV_NEED_PHY_INIT_FIX|DEV_NEED_MSI_FIX),
 	PCI_ROM(0x10DE, 0x0AB2, "MCP79", "MCP79 Ethernet Controller", DEV_NEED_LINKTIMER|DEV_HAS_LARGEDESC|DEV_HAS_CHECKSUM| DEV_HAS_HIGH_DMA|DEV_HAS_MSI|DEV_HAS_POWER_CNTRL| DEV_HAS_PAUSEFRAME_TX_V3|DEV_HAS_STATISTICS_V3| DEV_HAS_TEST_EXTENDED|DEV_HAS_CORRECT_MACADDR| DEV_HAS_COLLISION_FIX|DEV_NEED_TX_LIMIT2|DEV_HAS_GEAR_MODE| DEV_NEED_PHY_INIT_FIX|DEV_NEED_MSI_FIX),
diff --git a/src/drivers/net/intel.c b/src/drivers/net/intel.c
index 46527bdb..7879714f 100644
--- a/src/drivers/net/intel.c
+++ b/src/drivers/net/intel.c
@@ -1043,6 +1043,7 @@ static struct pci_device_id intel_nics[] = {
 	PCI_ROM ( 0x8086, 0x0d4f, "i219v-10", "I219-V (10)", INTEL_I219 ),
 	PCI_ROM ( 0x8086, 0x0d53, "i219lm-12", "I219-LM (12)", INTEL_I219 ),
 	PCI_ROM ( 0x8086, 0x0d55, "i219v-12", "I219-V (12)", INTEL_I219 ),
+	PCI_ROM ( 0x8086, 0x0dc5, "i219lm-23", "I219-LM (23)", INTEL_I219 ),
 	PCI_ROM ( 0x8086, 0x1000, "82542-f", "82542 (Fiber)", 0 ),
 	PCI_ROM ( 0x8086, 0x1001, "82543gc-f", "82543GC (Fiber)", 0 ),
 	PCI_ROM ( 0x8086, 0x1004, "82543gc", "82543GC (Copper)", 0 ),
diff --git a/src/drivers/net/iphone.c b/src/drivers/net/iphone.c
index 7d0eb4b6..bbac527b 100644
--- a/src/drivers/net/iphone.c
+++ b/src/drivers/net/iphone.c
@@ -1304,7 +1304,9 @@ ipair_tx ( struct ipair *ipair, const char *fmt, ... ) {
 	memset ( hdr, 0, sizeof ( *hdr ) );
 	hdr->len = htonl ( len );
 	msg = iob_put ( iobuf, len );
+	va_start ( args, fmt );
 	vsnprintf ( msg, len, fmt, args );
+	va_end ( args );
 	DBGC2 ( ipair, "IPAIR %p transmitting:\n%s\n", ipair, msg );
 
 	/* Transmit message */
diff --git a/src/drivers/net/marvell/aqc1xx.c b/src/drivers/net/marvell/aqc1xx.c
new file mode 100644
index 00000000..42b8164a
--- /dev/null
+++ b/src/drivers/net/marvell/aqc1xx.c
@@ -0,0 +1,643 @@
+/** @file
+ *
+ * Marvell AQtion family network card driver.
+ *
+ * Copyright(C) 2017-2021 Marvell
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS
+ *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ *  TO,THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ *   PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR HOLDER OR
+ *   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ *   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ *   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ *   OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ *   WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ *   OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ *   EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+FILE_LICENCE ( BSD2 );
+
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <byteswap.h>
+#include <ipxe/netdevice.h>
+#include <ipxe/ethernet.h>
+#include <ipxe/if_ether.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/malloc.h>
+#include <ipxe/pci.h>
+#include <ipxe/profile.h>
+
+#include "aqc1xx.h"
+
+extern struct atl_hw_ops atl_hw;
+extern struct atl_hw_ops atl2_hw;
+
+/** @file
+*
+* Marvell AQC network card driver
+*
+*/
+
+static int atl_ring_alloc ( const struct atl_nic *nic, struct atl_ring *ring,
+			    uint32_t desc_size, uint32_t reg_base )
+{
+	physaddr_t phy_addr;
+
+	/* Allocate ring buffer.*/
+	ring->length = ATL_RING_SIZE * desc_size;
+	ring->ring = dma_alloc ( nic->dma, &ring->map, ring->length,
+				ring->length );
+
+	if ( !ring->ring )
+		return -ENOMEM;
+
+	/* Initialize the descriptor ring */
+	memset ( ring->ring, 0, ring->length );
+
+	/* Program ring address */
+	phy_addr = dma ( &ring->map, ring->ring );
+
+	/* Write ring address (hi & low parts).*/
+	ATL_WRITE_REG ( (uint32_t)phy_addr, reg_base );
+	ATL_WRITE_REG ( (uint32_t)(((uint64_t)phy_addr) >> 32), reg_base + 4 );
+
+	/* Write ring length.*/
+	ATL_WRITE_REG ( ATL_RING_SIZE, reg_base + 8 );
+
+	ring->sw_head = ring->sw_tail = 0;
+
+	DBGC ( nic, "AQUANTIA: %p ring is at [%08llx,%08llx), reg base %#x\n",
+	    nic, ((unsigned long long)phy_addr),
+	    ((unsigned long long) phy_addr + ring->length), reg_base );
+
+	return 0;
+}
+
+static void atl_ring_free ( struct atl_ring *ring )
+{
+	dma_free ( &ring->map, ring->ring, ring->length );
+	ring->ring = NULL;
+	ring->length = 0;
+}
+
+static void atl_ring_next_dx ( unsigned int *val )
+{
+	++( *val );
+	if ( *val == ATL_RING_SIZE )
+		*val = 0;
+}
+
+int atl_ring_full ( const struct atl_ring *ring )
+{
+	unsigned int tail = ring->sw_tail;
+	atl_ring_next_dx ( &tail );
+	return tail == ring->sw_head;
+}
+
+void atl_rx_ring_fill ( struct atl_nic *nic )
+{
+	struct atl_desc_rx *rx;
+	struct io_buffer *iobuf;
+	physaddr_t address;
+	unsigned int refilled = 0;
+
+	/* Refill ring */
+	while ( !atl_ring_full ( &nic->rx_ring ) ) {
+
+		/* Allocate I/O buffer */
+		iobuf = alloc_rx_iob ( ATL_RX_MAX_LEN, nic->dma );
+		if ( !iobuf ) {
+			/* Wait for next refill */
+			break;
+		}
+
+		/* Get next receive descriptor */
+		rx = ( struct atl_desc_rx * )nic->rx_ring.ring +
+			nic->rx_ring.sw_tail;
+
+		/* Populate receive descriptor */
+		address = iob_dma ( iobuf );
+		rx->data_addr = address;
+		rx->hdr_addr = 0;
+
+		/* Record I/O buffer */
+		assert ( nic->iobufs[nic->rx_ring.sw_tail] == NULL );
+		nic->iobufs[nic->rx_ring.sw_tail] = iobuf;
+
+		DBGC( nic, "AQUANTIA: RX[%d] is [%llx,%llx)\n",
+			nic->rx_ring.sw_tail,
+			( (unsigned long long)address),
+			( (unsigned long long)address + ATL_RX_MAX_LEN) );
+
+		atl_ring_next_dx ( &nic->rx_ring.sw_tail );
+		refilled++;
+	}
+
+	/* Push descriptors to card, if applicable */
+	if ( refilled ) {
+		wmb();
+		ATL_WRITE_REG ( nic->rx_ring.sw_tail, ATL_RING_TAIL_PTR );
+	}
+}
+
+/**
+* Open network device
+*
+* @v netdev		Network device
+* @ret rc		Return status code
+*/
+static int atl_open ( struct net_device *netdev )
+{
+	struct atl_nic *nic = netdev->priv;
+	uint32_t ctrl = 0;
+
+	/* Tx ring */
+	if ( atl_ring_alloc ( nic, &nic->tx_ring, sizeof(struct atl_desc_tx),
+				ATL_TX_DMA_DESC_ADDR ) != 0 )
+		goto err_tx_alloc;
+
+	/* Rx ring */
+	if ( atl_ring_alloc ( nic, &nic->rx_ring, sizeof(struct atl_desc_rx),
+				ATL_RX_DMA_DESC_ADDR ) != 0 )
+		goto err_rx_alloc;
+
+	/* Allocate interrupt vectors */
+	ATL_WRITE_REG ( (ATL_IRQ_CTRL_COR_EN | ATL_IRQ_CTRL_REG_RST_DIS),
+		      ATL_IRQ_CTRL );
+
+	/*TX & RX Interruprt Mapping*/
+	ctrl = ATL_IRQ_MAP_REG1_RX0 | ATL_IRQ_MAP_REG1_RX0_EN |
+		   ATL_IRQ_MAP_REG1_TX0 | ATL_IRQ_MAP_REG1_TX0_EN;
+	ATL_WRITE_REG ( ctrl, ATL_IRQ_MAP_REG1 );
+
+	/*TX interrupt ctrl reg*/
+	ATL_WRITE_REG ( ATL_TX_IRQ_CTRL_WB_EN, ATL_TX_IRQ_CTRL );
+
+	/*RX interrupt ctrl reg*/
+	ATL_WRITE_REG ( ATL_RX_IRQ_CTRL_WB_EN, ATL_RX_IRQ_CTRL );
+
+	/*RX data path*/
+	ctrl = ATL_IRQ_TX | ATL_IRQ_RX;
+	/* itr mask */
+	ATL_WRITE_REG ( ctrl,  ATL_ITR_MSKS );
+	ATL_WRITE_REG ( (uint32_t)ATL_RX_MAX_LEN / 1024U,
+		      ATL_RX_DMA_DESC_BUF_SIZE );
+
+	/*filter global ctrl */
+	ctrl = ATL_RPF_CTRL1_BRC_EN | ATL_RPF_CTRL1_L2_PROMISC |
+		ATL_RPF_CTRL1_ACTION | ATL_RPF_CTRL1_BRC_TSH;
+	ATL_WRITE_REG ( ctrl, ATL_RPF_CTRL1 );
+
+	/* vlan promisc */
+	ATL_WRITE_REG ( ATL_RPF_CTRL2_VLAN_PROMISC, ATL_RPF_CTRL2 );
+	/* enable rpf2 */
+	ATL_WRITE_REG ( ATL_RPF2_CTRL_EN, ATL_RPF2_CTRL );
+
+	/* RX Packet Buffer 0 Register 1 */
+	ATL_WRITE_REG ( ATL_RPB0_CTRL1_SIZE, ATL_RPB0_CTRL1 );
+
+	/*RX Packet Buffer 0 Register 2 */
+	ctrl = ATL_RPB0_CTRL2_LOW_TSH | ATL_RPB0_CTRL2_HIGH_TSH |
+		ATL_RPB0_CTRL2_FC_EN;
+	ATL_WRITE_REG ( ctrl, ATL_RPB0_CTRL2 );
+
+	/*RPB global ctrl*/
+	ctrl = ATL_READ_REG(ATL_RPB_CTRL);
+	ctrl |= (ATL_RPB_CTRL_EN | ATL_RPB_CTRL_FC);
+	ATL_WRITE_REG ( ctrl, ATL_RPB_CTRL );
+
+	/*TX data path*/
+	/* enable tpo2 */
+	ATL_WRITE_REG ( ATL_TPO2_EN, ATL_TPO2_CTRL );
+	/* tpb global ctrl *** */
+	ATL_WRITE_REG ( ATL_TPB0_CTRL1_SIZE, ATL_TPB0_CTRL1 );
+
+	ctrl = ATL_TPB0_CTRL2_LOW_TSH | ATL_TPB0_CTRL2_HIGH_TSH;
+	/* tpb global ctrl *** */
+	ATL_WRITE_REG ( ctrl, ATL_TPB0_CTRL2 );
+
+	ctrl = ATL_READ_REG ( ATL_TPB_CTRL );
+	ctrl |= ( ATL_TPB_CTRL_EN | ATL_TPB_CTRL_PAD_EN );
+	/* tpb global ctrl */
+	ATL_WRITE_REG ( ctrl, ATL_TPB_CTRL );
+
+	/*Enable rings*/
+	ATL_WRITE_REG ( ATL_READ_REG ( ATL_RING_TX_CTRL ) | ATL_RING_TX_CTRL_EN,
+		      ATL_RING_TX_CTRL );
+	ATL_WRITE_REG ( ATL_READ_REG ( ATL_RING_RX_CTRL ) | ATL_RING_RX_CTRL_EN,
+		      ATL_RING_RX_CTRL );
+
+	if ( nic->flags == ATL_FLAG_A2 ) {
+		ATL_WRITE_REG ( ATL2_RPF_NEW_EN_ADR_EN, ATL2_RPF_NEW_EN_ADR );
+	}
+
+	atl_rx_ring_fill ( nic );
+
+	nic->hw_ops->start ( nic );
+
+	return 0;
+
+err_rx_alloc:
+	atl_ring_free ( &nic->tx_ring );
+
+err_tx_alloc:
+	return -ENOMEM;
+}
+
+/**
+* Close network device
+*
+* @v netdev		Network device
+*/
+static void atl_close ( struct net_device *netdev )
+{
+	struct atl_nic *nic = netdev->priv;
+
+	nic->hw_ops->stop ( nic );
+	/* rpb global ctrl */
+	ATL_WRITE_REG ( ATL_RPB_CTRL_DIS, ATL_RPB_CTRL );
+	/* tgb global ctrl */
+	ATL_WRITE_REG ( ATL_TPB_CTRL_DIS, ATL_TPB_CTRL);
+
+	ATL_WRITE_REG ( ATL_READ_REG(ATL_RING_TX_CTRL) | (~ATL_RING_TX_CTRL_EN),
+			ATL_RING_TX_CTRL );
+	ATL_WRITE_REG ( ATL_READ_REG(ATL_RING_RX_CTRL) | (~ATL_RING_RX_CTRL_EN),
+			ATL_RING_RX_CTRL );
+
+	/* clear itr mask */
+	ATL_WRITE_REG ( ATL_ITR_MSKS_DIS, ATL_ITR_MSKS );
+
+	/* Reset the NIC */
+	nic->hw_ops->reset ( nic );
+
+	atl_ring_free ( &nic->tx_ring );
+	atl_ring_free ( &nic->rx_ring );
+}
+
+/**
+* Transmit packet
+*
+* @v netdev		Network device
+* @v iobuf		I/O buffer
+* @ret rc		Return status code
+*/
+int atl_transmit ( struct net_device *netdev, struct io_buffer *iobuf )
+{
+	struct atl_nic *nic = netdev->priv;
+	struct atl_desc_tx *tx;
+	physaddr_t address;
+	uint32_t len;
+
+	/* Get next transmit descriptor */
+	if ( atl_ring_full ( &nic->tx_ring ) ) {
+		DBGC ( nic, "AQUANTIA: %p out of transmit descriptors\n", nic );
+		return -ENOBUFS;
+	}
+
+	tx = (struct atl_desc_tx *)nic->tx_ring.ring + nic->tx_ring.sw_tail;
+
+	/* Populate transmit descriptor */
+	memset ( tx, 0, sizeof ( *tx ) );
+	address = iob_dma ( iobuf );
+	tx->address = address;
+	len = iob_len ( iobuf );
+
+	tx->status = 0x1;
+	tx->status = ( (tx->status) & ~ATL_DESC_TX_BUF_LEN_MASK) | 
+			((len << ATL_DESC_TX_BUF_LEN_OFFSET) & 
+			ATL_DESC_TX_BUF_LEN_MASK ); 
+	tx->status = ((tx->status) & ~ATL_DESC_TX_EOP_MASK) | 
+			( (ATL_DESC_TX_DX_EOP_VALUE << ATL_DESC_TX_EOP_OFFSET) &
+			ATL_DESC_TX_EOP_MASK );
+	tx->status = ( (tx->status) & ~ATL_DESC_TX_CMD_MASK) | 
+			((ATL_DESC_TX_CMD_VALUE << ATL_DESC_TX_CMD_OFFSET) &
+			ATL_DESC_TX_CMD_MASK ); 
+	tx->flag = ( (tx->flag) & ~ATL_DESC_TX_PAY_LEN_MASK) | 
+			((len << ATL_DESC_TX_PAY_LEN_OFFSET) &
+			ATL_DESC_TX_PAY_LEN_MASK );
+	wmb();
+
+	DBGC2 ( nic, "AQUANTIA: %p TX[%d] is [%llx, %llx]\n",
+		nic, nic->tx_ring.sw_tail,
+	    ( ( unsigned long long ) address ),
+	    ( ( unsigned long long ) address + len ) );
+
+	atl_ring_next_dx ( &nic->tx_ring.sw_tail );
+	ATL_WRITE_REG ( nic->tx_ring.sw_tail, ATL_RING_TAIL );
+
+	return 0;
+}
+
+void atl_check_link ( struct net_device *netdev )
+{
+	struct atl_nic *nic = netdev->priv;
+	uint32_t link_state;
+
+	/* Read link status */
+	link_state = nic->hw_ops->get_link ( nic );
+
+	DBGC ( nic, "AQUANTIA: %p link status is %08x\n", nic, link_state );
+
+	if ( link_state != nic->link_state ) {
+		if ( link_state ) {
+			DBGC ( nic, "AQUANTIA: link up\n");
+			netdev_link_up ( netdev );
+		} else {
+			DBGC ( nic, "AQUANTIA: link lost\n");
+			netdev_link_down ( netdev );
+		}
+		nic->link_state = link_state;
+	}
+}
+
+/**
+* Poll for completed packets
+*
+* @v netdev		Network device
+*/
+void atl_poll_tx ( struct net_device *netdev )
+{
+	struct atl_nic *nic = netdev->priv;
+	struct atl_desc_tx_wb *tx;
+
+	/* Check for completed packets */
+	while ( nic->tx_ring.sw_head != nic->tx_ring.sw_tail ) {
+
+		/* Get next transmit descriptor */
+		tx = ( struct atl_desc_tx_wb * )nic->tx_ring.ring +
+		nic->tx_ring.sw_head;
+
+		/* Stop if descriptor is still in use */
+		if ( !(tx->status & cpu_to_le32 ( ATL_TX_DESC_STATUS_DD ) ) )
+			return;
+
+		DBGC2 ( nic, "AQUANTIA: %p TX[%d] complete\n",
+			nic, nic->tx_ring.sw_head );
+
+		/* Complete TX descriptor */
+		atl_ring_next_dx ( &nic->tx_ring.sw_head );
+		netdev_tx_complete_next ( netdev );
+	}
+}
+
+/**
+* Poll for received packets
+*
+* @v netdev		Network device
+*/
+void atl_poll_rx ( struct net_device *netdev )
+{
+	struct atl_nic *nic = netdev->priv;
+	struct atl_desc_rx_wb *rx;
+	struct io_buffer *iobuf;
+	size_t len;
+
+	/* Check for received packets */
+	while ( nic->rx_ring.sw_head != nic->rx_ring.sw_tail ) {
+
+		/* Get next receive descriptor */
+		rx = (struct atl_desc_rx_wb *)nic->rx_ring.ring +
+		nic->rx_ring.sw_head;
+
+		/* Stop if descriptor is still in use */
+		if ( !(rx->status & cpu_to_le16(ATL_RX_DESC_STATUS_DD)) )
+			return;
+
+		/* Populate I/O buffer */
+		iobuf = nic->iobufs[nic->rx_ring.sw_head];
+		nic->iobufs[nic->rx_ring.sw_head] = NULL;
+		len = le16_to_cpu ( rx->pkt_len );
+		iob_put ( iobuf, len );
+
+		/* Hand off to network stack */
+		DBGC ( nic, "AQUANTIA: %p RX[%d] complete (length %zd)\n",
+		    nic, nic->rx_ring.sw_head, len );
+
+		netdev_rx ( netdev, iobuf );
+
+		atl_ring_next_dx ( &nic->rx_ring.sw_head );
+	}
+}
+
+/**
+* Poll for completed and received packets
+*
+* @v netdev		Network device
+*/
+static void atl_poll ( struct net_device *netdev )
+{
+	struct atl_nic *nic = netdev->priv;
+
+	/* Check link state */
+	atl_check_link ( netdev );
+
+	/* Poll for TX completions */
+	atl_poll_tx ( netdev );
+
+	/* Poll for RX completions */
+	atl_poll_rx ( netdev );
+
+	/* Refill RX ring */
+	atl_rx_ring_fill ( nic );
+}
+
+/**
+* Enable or disable interrupts
+*
+* @v netdev		Network device
+* @v enable		Interrupts should be enabled
+*/
+static void atl_irq ( struct net_device *netdev, int enable )
+{
+	struct atl_nic *nic = netdev->priv;
+	uint32_t mask;
+
+	mask = ( ATL_IRQ_TX | ATL_IRQ_RX );
+	if ( enable )
+		ATL_WRITE_REG ( mask, ATL_ITR_MSKS );
+	else
+		ATL_WRITE_REG ( mask, ATL_ITR_MSKC );
+}
+
+/** Marvell network device operations */
+static struct net_device_operations atl_operations = {
+	.open = atl_open,
+	.close = atl_close,
+	.transmit = atl_transmit,
+	.poll = atl_poll,
+	.irq = atl_irq,
+};
+
+/******************************************************************************
+*
+* PCI interface
+*
+*******************************************************************************
+*/
+
+/**
+* Probe PCI device
+*
+* @v pci		PCI device
+* @ret rc		Return status code
+*/
+static int atl_probe ( struct pci_device *pci )
+{
+	struct net_device *netdev;
+	struct atl_nic *nic;
+	int rc = ENOERR;
+	uint32_t io_size = 0;
+
+	/* Allocate and initialise net device */
+	netdev = alloc_etherdev ( sizeof( *nic ) );
+	if ( !netdev ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	netdev_init ( netdev, &atl_operations );
+	nic = netdev->priv;
+	pci_set_drvdata ( pci, netdev );
+	netdev->dev = &pci->dev;
+	memset( nic, 0, sizeof( *nic ) );
+	nic->flags = pci->id->driver_data;
+
+	/* Fix up PCI device */
+	adjust_pci_device ( pci );
+
+	switch ( nic->flags ) {
+		case ATL_FLAG_A1:
+			nic->hw_ops = &atl_hw;
+			io_size = ATL_BAR_SIZE;
+			break;
+		case ATL_FLAG_A2:
+			nic->hw_ops = &atl2_hw;
+			io_size = ATL2_BAR_SIZE;
+			break;
+		default:
+			goto err_unsupported;
+			break;
+	}
+
+	/* Map registers */
+	nic->regs = pci_ioremap ( pci, pci->membase, io_size );
+	if ( !nic->regs ) {
+		rc = -ENODEV;
+		goto err_ioremap;
+	}
+
+	 /* Configure DMA */
+	nic->dma = &pci->dma;
+
+	/* Reset the NIC */
+	if ( ( rc = nic->hw_ops->reset ( nic ) ) != 0 )
+		goto err_reset;
+
+	/* Get MAC Address */
+	if ( ( rc = nic->hw_ops->get_mac ( nic, netdev->hw_addr ) ) != 0 )
+		goto err_mac;
+
+	/* Register network device */
+	if ( ( rc = register_netdev ( netdev ) ) != 0 )
+		goto err_register_netdev;
+
+	/* Set initial link state */
+	netdev_link_down ( netdev );
+
+	return 0;
+
+err_register_netdev:
+err_mac:
+	nic->hw_ops->reset ( nic );
+err_reset:
+	iounmap ( nic->regs );
+err_ioremap:
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+err_unsupported:
+err_alloc:
+	return rc;
+}
+
+/**
+* Remove PCI device
+*
+* @v pci		PCI device
+*/
+static void atl_remove ( struct pci_device *pci )
+{
+	struct net_device *netdev = pci_get_drvdata ( pci );
+	struct atl_nic *nic = netdev->priv;
+
+	/* Unregister network device */
+	unregister_netdev ( netdev );
+
+	/* Reset the NIC */
+	nic->hw_ops->reset ( nic );
+
+	/* Free network device */
+	iounmap ( nic->regs );
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+}
+
+/** Marvell PCI device IDs */
+static struct pci_device_id atl_nics[] = {
+	/* Atlantic 1 */
+	/* 10G */
+	PCI_ROM(0x1D6A, 0x0001, "AQC07", "Marvell AQtion 10Gbit Network Adapter", ATL_FLAG_A1),
+	PCI_ROM(0x1D6A, 0xD107, "AQC07", "Marvell AQtion 10Gbit Network Adapter", ATL_FLAG_A1),
+	PCI_ROM(0x1D6A, 0x07B1, "AQC07", "Marvell AQtion 10Gbit Network Adapter", ATL_FLAG_A1),
+	PCI_ROM(0x1D6A, 0x87B1, "AQC07", "Marvell AQtion 10Gbit Network Adapter", ATL_FLAG_A1),
+
+	/* SFP */
+	PCI_ROM(0x1D6A, 0xD100, "AQC00", "Felicity Network Adapter", ATL_FLAG_A1),
+	PCI_ROM(0x1D6A, 0x00B1, "AQC00", "Felicity Network Adapter", ATL_FLAG_A1),
+	PCI_ROM(0x1D6A, 0x80B1, "AQC00", "Felicity Network Adapter", ATL_FLAG_A1),
+
+	/* 5G */
+	PCI_ROM(0x1D6A, 0xD108, "AQC08", "Marvell AQtion 5Gbit Network Adapter", ATL_FLAG_A1),
+	PCI_ROM(0x1D6A, 0x08B1, "AQC08", "Marvell AQtion 5Gbit Network Adapter", ATL_FLAG_A1),
+	PCI_ROM(0x1D6A, 0x88B1, "AQC08", "Marvell AQtion 5Gbit Network Adapter", ATL_FLAG_A1),
+	PCI_ROM(0x1D6A, 0x11B1, "AQC11", "Marvell AQtion 5Gbit Network Adapter", ATL_FLAG_A1),
+	PCI_ROM(0x1D6A, 0x91B1, "AQC11", "Marvell AQtion 5Gbit Network Adapter", ATL_FLAG_A1),
+
+	/* 2.5G */
+	PCI_ROM(0x1D6A, 0xD109, "AQC09", "Marvell AQtion 2.5Gbit Network Adapter", ATL_FLAG_A1),
+	PCI_ROM(0x1D6A, 0x09B1, "AQC09", "Marvell AQtion 2.5Gbit Network Adapter", ATL_FLAG_A1),
+	PCI_ROM(0x1D6A, 0x89B1, "AQC09", "Marvell AQtion 2.5Gbit Network Adapter", ATL_FLAG_A1),
+	PCI_ROM(0x1D6A, 0x12B1, "AQC12", "Marvell AQtion 2.5Gbit Network Adapter", ATL_FLAG_A1),
+	PCI_ROM(0x1D6A, 0x92B1, "AQC12", "Marvell AQtion 2.5Gbit Network Adapter", ATL_FLAG_A1),
+
+	/* Atlantic 2 */
+	PCI_ROM(0x1D6A, 0x00C0, "AQC13", "Marvell AQtion 10Gbit Network Adapter", ATL_FLAG_A2),
+	PCI_ROM(0x1D6A, 0x94C0, "AQC13", "Marvell AQtion 10Gbit Network Adapter", ATL_FLAG_A2),
+	PCI_ROM(0x1D6A, 0x93C0, "AQC13", "Marvell AQtion 10Gbit Network Adapter", ATL_FLAG_A2),
+	PCI_ROM(0x1D6A, 0x04C0, "AQC13", "Marvell AQtion 10Gbit Network Adapter", ATL_FLAG_A2),
+	PCI_ROM(0x1D6A, 0x14C0, "AQC13", "Marvell AQtion 10Gbit Network Adapter", ATL_FLAG_A2),
+	PCI_ROM(0x1D6A, 0x12C0, "AQC13", "Marvell AQtion 10Gbit Network Adapter", ATL_FLAG_A2),
+};
+
+/** Marvell PCI driver */
+struct pci_driver atl_driver __pci_driver = {
+	.ids = atl_nics,
+	.id_count = (sizeof(atl_nics) / sizeof(atl_nics[0])),
+	.probe = atl_probe,
+	.remove = atl_remove,
+};
\ No newline at end of file
diff --git a/src/drivers/net/marvell/aqc1xx.h b/src/drivers/net/marvell/aqc1xx.h
new file mode 100644
index 00000000..c3e34e1e
--- /dev/null
+++ b/src/drivers/net/marvell/aqc1xx.h
@@ -0,0 +1,270 @@
+/** @file
+ *
+ * Marvell AQtion family network card driver definitions.
+ *
+ * Copyright(C) 2017-2021 Marvell
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS
+ *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ *  TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ *   PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR HOLDER OR
+ *   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ *   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ *   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ *   OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ *   WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ *   OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ *   EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#ifndef _ATLANTIC_H
+#define _ATLANTIC_H
+
+FILE_LICENCE ( BSD2 );
+
+#include <stdint.h>
+#include <ipxe/if_ether.h>
+#include <ipxe/nvs.h>
+
+#define ATL_BAR_SIZE   0x10000
+#define ATL2_BAR_SIZE   0x40000
+#define ATL_RING_SIZE  64
+#define ATL_RING_ALIGN 128
+#define ATL_RX_MAX_LEN 2048
+
+#define ATL_IRQ_TX   0x00000001U
+#define ATL_IRQ_RX   0x00000002U
+
+/*IRQ Status Register*/
+#define ATL_IRQ_STAT_REG 0x00002000U
+
+/* Interrupt Vector Allocation Register */
+#define ATL_IRQ_CTRL			 0x00002300U
+#define ATL_IRQ_CTRL_COR_EN	  0x00000080U /*IRQ clear on read */
+#define ATL_IRQ_CTRL_REG_RST_DIS 0x20000000U /*Register reset disable */
+
+/*TX/RX Interruprt Mapping*/
+#define ATL_IRQ_MAP_REG1		0x00002100U /*IRQ mapping register */
+
+#define ATL_IRQ_MAP_REG1_RX0_EN 0x00008000U /*IRQ RX0 enable*/
+#define ATL_IRQ_MAP_REG1_RX0	0x00000100U /*IRQ RX0*/
+
+#define ATL_IRQ_MAP_REG1_TX0_EN 0x80000000U /*IRQ TX0 enable*/
+#define ATL_IRQ_MAP_REG1_TX0	0x00000000U /*IRQ TX0*/
+
+/*TX interrupt ctrl reg*/
+#define ATL_TX_IRQ_CTRL	   0x00007B40U
+#define ATL_TX_IRQ_CTRL_WB_EN 0x00000002U
+
+/*RX interrupt ctrl reg*/
+#define ATL_RX_IRQ_CTRL	   0x00005A30U
+#define ATL_RX_IRQ_CTRL_WB_EN 0x00000004U
+
+#define ATL_GLB_CTRL  0x00000000U
+
+#define ATL_PCI_CTRL		 0x00001000U
+#define ATL_PCI_CTRL_RST_DIS 0x20000000U
+
+#define ATL_RX_CTRL		 0x00005000U
+#define ATL_RX_CTRL_RST_DIS 0x20000000U /*RPB reset disable */
+#define ATL_TX_CTRL		 0x00007000U
+#define ATL_TX_CTRL_RST_DIS 0x20000000U /*TPB reset disable */
+
+/*RX data path control registers*/
+#define ATL_RPF2_CTRL	0x00005040U
+#define ATL_RPF2_CTRL_EN 0x000F0000U /* RPF2 enable*/
+#define ATL2_RPF_NEW_EN_ADR_EN	  0x00000001U /*enable*/
+#define ATL2_RPF_NEW_EN_ADR 0x5104
+
+#define ATL_RPF_CTRL1		0x00005100U
+#define ATL_RPF_CTRL1_BRC_EN	 0x00000001U /*Allow broadcast receive*/
+#define ATL_RPF_CTRL1_L2_PROMISC 0x00000008U /*L2 promiscious*/
+#define ATL_RPF_CTRL1_ACTION	 0x00001000U /*Action to host*/
+#define ATL_RPF_CTRL1_BRC_TSH	0x00010000U /*Brc threshold 256 units per sec*/
+
+#define ATL_RPF_CTRL2		0x00005280U
+#define ATL_RPF_CTRL2_VLAN_PROMISC 0x00000002U /*VLAN promisc*/
+
+#define ATL_RPB_CTRL_DIS      0x0
+#define ATL_RPB_CTRL		 0x00005700U
+#define ATL_RPB_CTRL_EN	  0x00000001U /*RPB Enable*/
+#define ATL_RPB_CTRL_FC	  0x00000010U /*RPB Enable*/
+#define ATL_RPB_CTRL_TC_MODE 0x00000100U /*RPB Traffic Class Mode*/
+
+#define ATL_RPB0_CTRL1	  0x00005710U
+#define ATL_RPB0_CTRL1_SIZE 0x00000140U /*RPB size (in unit 1KB) \*/
+
+#define ATL_RPB0_CTRL2		0x00005714U
+
+/*Buffer Low Threshold (70% of RPB size in unit 32B)*/
+#define ATL_RPB0_CTRL2_LOW_TSH  0x00000C00U
+/*Buffer High Threshold(30% of RPB size in unit 32B)*/
+#define ATL_RPB0_CTRL2_HIGH_TSH 0x1C000000U
+#define ATL_RPB0_CTRL2_FC_EN	0x80000000U /*Flow control Enable*/
+
+#define ATL_RX_DMA_DESC_BUF_SIZE 0x00005b18U
+#define ATL_RX_DMA_DESC_ADDR 0x00005b00U
+
+/*TX data path  control registers*/
+#define ATL_TPO2_CTRL 0x00007040U
+#define ATL_TPO2_EN   0x00010000U /*TPO2 Enable*/
+
+#define ATL_TPB_CTRL_DIS      0x0
+#define ATL_TPB_CTRL		 0x00007900U
+#define ATL_TPB_CTRL_EN	  0x00000001U /*TPB enable*/
+#define ATL_TPB_CTRL_PAD_EN  0x00000004U /*Tx pad insert enable*/
+#define ATL_TPB_CTRL_TC_MODE 0x00000100U /*Tx traffic Class Mode*/
+
+#define ATL_TPB0_CTRL1	  0x00007910U
+#define ATL_TPB0_CTRL1_SIZE 0x000000A0U /*TPB Size (in unit 1KB)*/
+
+#define ATL_TPB0_CTRL2		  0x00007914U
+/*Buffer Low  Threshold(30% of RPB size in unit 32B)*/
+#define ATL_TPB0_CTRL2_LOW_TSH  0x00000600U
+/*Buffer High  Threshold(30% of RPB size in unit 32B)*/
+#define ATL_TPB0_CTRL2_HIGH_TSH 0x0E000000U
+
+#define ATL_TX_DMA_DESC_ADDR 0x00007c00U
+
+/*Rings control registers*/
+#define ATL_RING_TX_CTRL	0x00007c08U
+#define ATL_RING_TX_CTRL_EN 0x80000000U /*Tx descriptor Enable*/
+
+#define ATL_RING_RX_CTRL	0x00005b08U
+#define ATL_RING_RX_CTRL_EN 0x80000000U /*Rx descriptor Enable*/
+
+#define ATL_RING_TAIL	 0x00007c10U
+#define ATL_RING_TAIL_PTR 0x00005b10U
+
+/*IRQ control registers*/
+#define ATL_ITR_MSKS_DIS      0x0
+#define ATL_ITR_MSKS	 0x00002060U
+#define ATL_ITR_MSKS_LSW 0x0000000CU
+#define ATL_ITR_MSKC	 0x00002070U
+#define ATL_ITR_MSKC_LSW 0x0000000CU
+
+/*Link advertising*/
+#define ATL_LINK_ADV		   0x00000368U
+#define ATL_SHUT_LINK      0x0
+#define ATL_LINK_ADV_AUTONEG   0xF20U
+
+#define ATL_LINK_ST	 0x00000370U
+
+/*Semaphores*/
+#define ATL_SEM_RAM 0x000003a8U
+#define ATL_SEM_RAM_RESET 0X1
+
+/*Mailbox*/
+#define ATL_MBOX_ADDR  0x00000360U
+#define ATL_MBOX_CTRL1 0x00000200U
+#define ATL_MBOX_CTRL1_START_MBOX_OPT 0x8000
+
+#define ATL_MBOX_CTRL3 0x00000208U
+#define ATL_MBOX_CTRL5 0x0000020cU
+
+#define ATL_FLAG_A1 0x1
+#define ATL_FLAG_A2 0x2
+
+/*write register*/
+#define ATL_WRITE_REG(VAL, REG)	writel(VAL, nic->regs + (REG))
+#define ATL_READ_REG(REG)	readl(nic->regs + (REG)) /*read register*/
+
+struct atl_desc_tx {
+	uint64_t address;
+	uint32_t status;
+	uint32_t flag;
+} __attribute__((packed));
+
+#define ATL_DESC_TX_DX_TYPE_VALUE 0x1
+
+#define  ATL_DESC_TX_DX_EOP_VALUE  0x1
+#define  ATL_DESC_TX_EOP_MASK     0x00200000
+#define  ATL_DESC_TX_EOP_OFFSET   21
+
+#define ATL_DESC_TX_CMD_MASK    0x3FC00000UL
+#define ATL_DESC_TX_CMD_OFFSET  22
+#define ATL_DESC_TX_CMD_VALUE   0x22
+
+#define ATL_DESC_TX_BUF_LEN_MASK   0x000FFFF0
+#define ATL_DESC_TX_BUF_LEN_OFFSET 5
+
+#define ATL_DESC_TX_PAY_LEN_MASK   0xFFFFC000
+#define ATL_DESC_TX_PAY_LEN_OFFSET 14
+
+struct atl_desc_tx_wb {
+	uint64_t rsvd1;
+	uint32_t status;
+	uint32_t rsvd4;
+} __attribute__((packed));
+
+#define ATL_TX_DESC_STATUS_DD  0x00100000UL
+
+struct atl_desc_rx {
+	uint64_t data_addr;
+	uint64_t hdr_addr;
+
+} __attribute__((packed));
+
+struct atl_desc_rx_wb {
+	uint64_t rsvd2;
+	uint16_t status;
+	uint16_t pkt_len;
+	uint32_t rsvd4;
+} __attribute__((packed));
+
+#define ATL_RX_DESC_STATUS_DD  0x0001UL
+#define ATL_RX_DESC_STATUS_EOP 0x0002UL
+struct atl_ring {
+	unsigned int sw_tail;
+	unsigned int sw_head;
+	void *ring;
+	/** Descriptor ring DMA mapping */
+	struct dma_mapping map;
+	unsigned int length;
+};
+
+struct atl_nic;
+
+struct atl_hw_ops {
+	int (*reset) (struct atl_nic *nic);
+	int (*start) (struct atl_nic *nic);
+	int (*stop) (struct atl_nic *nic);
+	int (*get_link) (struct atl_nic *nic);
+	int (*get_mac) (struct atl_nic *, uint8_t *mac);
+};
+
+/** An aQuanita network card */
+struct atl_nic {
+	/** Registers */
+	void *regs;
+	/** Port number (for multi-port devices) */
+	unsigned int port;
+	/** DMA device */
+	struct dma_device *dma;
+	/** Flags */
+	unsigned int flags;
+	struct atl_ring tx_ring;
+	struct atl_ring rx_ring;
+	struct io_buffer *iobufs[ATL_RING_SIZE];
+	uint32_t link_state;
+	uint32_t mbox_addr;
+	struct atl_hw_ops *hw_ops;
+};
+
+struct atl_hw_stats {
+	uint32_t version;
+	uint32_t tid;
+};
+
+#endif /* _AQUANTIA_H */
diff --git a/src/drivers/net/marvell/atl2_hw.c b/src/drivers/net/marvell/atl2_hw.c
new file mode 100644
index 00000000..0c57a12f
--- /dev/null
+++ b/src/drivers/net/marvell/atl2_hw.c
@@ -0,0 +1,235 @@
+/** @file
+ *
+ * Marvell AQtion family network card driver, hardware-specific functions.
+ *
+ * Copyright(C) 2017-2021 Marvell
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ 
+ *  1. Redistributions of source code must retain the above copyright notice, 
+ *     this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright notice, 
+ *     this list of conditions and the following disclaimer in the documentation 
+ *     and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS 
+ *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ *  TO,THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ *   PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR HOLDER OR
+ *   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ *   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ *   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ *   OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ *   WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ *   OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ *   EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+FILE_LICENCE ( BSD2 );
+
+#include <errno.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <byteswap.h>
+#include <ipxe/pci.h>
+#include "aqc1xx.h"
+#include "atl2_hw.h"
+
+static int atl2_hw_boot_completed_ ( struct atl_nic *nic )
+{
+	uint32_t reset_status = ATL_READ_REG ( ATL2_GLB_RST_CTRL2 );
+
+	return ( reset_status & ATL2_RESET_STATUS_BOOT_COMPLETED_MASK ) ||
+		   ( ATL_READ_REG ( ATL2_HOST_ITR_REQ )
+		   & ATL2_FW_HOST_INTERRUPT_REQUEST_READY );
+}
+
+void atl2_hw_read_shared_in_ ( struct atl_nic *nic, uint32_t offset,
+								uint32_t *data, uint32_t len )
+{
+	uint32_t i;
+
+	for (i = 0; i < len; ++i )
+	{
+		data[i] = ATL_READ_REG ( ATL2_MIF_SHARED_BUF_IN + offset + i * 4 );
+	}
+}
+
+void atl2_hw_write_shared_in_ ( struct atl_nic *nic, uint32_t offset,
+							  uint32_t *data, uint32_t len )
+{
+	uint32_t i;
+
+	for ( i = 0; i < len; ++i )
+	{
+		ATL_WRITE_REG ( data[i], ATL2_MIF_SHARED_BUF_IN + offset + i * 4 );
+	}
+}
+
+int atl2_hw_finish_ack_ ( struct atl_nic *nic, uint32_t ms )
+{
+	uint32_t i;
+	int err = 0;
+
+	ATL_WRITE_REG ( ATL_READ_REG(ATL2_HOST_FINISHED_WRITE )
+	| 1, ATL2_HOST_FINISHED_WRITE );
+
+	for ( i = 0; i < (ms / 100); ++i )
+	{
+		if ( ( ATL_READ_REG(ATL2_MCP_BUSY_WRITE ) & 1 ) == 0 )
+		{
+			break;
+		}
+		udelay ( ATL2_DELAY_100 );
+	}
+	if (i == ( ms / 100 ) )
+		err = -ETIME;
+
+	return err;
+}
+
+int atl2_hw_fw_init_ ( struct atl_nic *nic )
+{
+	uint32_t val;
+	int err = 0;
+
+	atl2_hw_read_shared_in_ ( nic, ATL2_LINK_CTRL_IN_OFF, &val, 1 );
+	val |= ( ATL2_HOST_MODE_ACTIVE | ( 1U << 13 ) );
+	atl2_hw_write_shared_in_ ( nic, ATL2_LINK_CTRL_IN_OFF, &val, 1 );
+
+	atl2_hw_read_shared_in_ ( nic, ATL2_MTU_IN_OFF, &val, 1 );
+	val = 16352;
+	atl2_hw_write_shared_in_ ( nic, ATL2_MTU_IN_OFF, &val, 1 );
+
+	atl2_hw_read_shared_in_ ( nic, ATL2_LINK_OPTS_IN_OFF, &val, 1 );
+	val = 0;
+	atl2_hw_write_shared_in_( nic, ATL2_LINK_OPTS_IN_OFF, &val, 1 );
+	err = atl2_hw_finish_ack_ ( nic, 50000000 );
+
+	return err;
+}
+
+int atl2_hw_reset ( struct atl_nic *nic )
+{
+	int completed = 0;
+	uint32_t status = 0;
+	uint32_t request;
+	int err = 0;
+	int i;
+
+	request = ATL2_RESET_STATUS_REQ_GSR;
+
+	ATL_WRITE_REG ( request, ATL2_GLB_RST_CTRL2 );
+
+	/* Wait for boot code started every 10us, 200 ms */
+	for ( i = 0; i < 20000; ++i )
+	{
+		status = ATL_READ_REG ( ATL2_GLB_RST_CTRL2 );
+
+		if ( ( ( status & ATL2_RESET_STATUS_BC_STARTED ) &&
+			 (status != 0xFFFFFFFFu ) ) )
+			break;
+
+		udelay ( ATL2_DELAY_10 );
+	}
+	if ( i == 20000 )
+	{
+		DBGC ( nic, "Boot code hanged" );
+		err = -EIO;
+		goto err_exit;
+	}
+
+	/* Wait for boot succeed, failed or host request every 10us, 480ms */
+	for ( i = 0; i < 48000; ++i )
+	{
+		completed = atl2_hw_boot_completed_ ( nic );
+		if ( completed )
+			break;
+
+		udelay ( ATL2_DELAY_10 );
+	}
+
+	if ( !completed )
+	{
+		DBGC ( nic, "FW Restart timed out" );
+		err = -ETIME;
+		goto err_exit;
+	}
+
+	status = ATL_READ_REG ( ATL2_GLB_RST_CTRL2 );
+
+	if ( status & ATL2_RESET_STATUS_BOOT_FAILED_MASK )
+	{
+		err = -EIO;
+		DBGC ( nic, "FW Restart failed" );
+		DBGC ( nic, "status = 0x%x", status );
+		goto err_exit;
+	}
+
+	if ( ATL_READ_REG ( ATL2_HOST_ITR_REQ )
+			& ATL2_FW_HOST_INTERRUPT_REQUEST_READY )
+	{
+		err = -ENOTSUP;
+		DBGC ( nic, "Dynamic FW load not implemented" );
+		goto err_exit;
+	}
+
+	err = atl2_hw_fw_init_ ( nic );
+
+err_exit:
+	return err;
+}
+
+int atl2_hw_start ( struct atl_nic *nic )
+{
+	uint32_t val;
+
+	atl2_hw_read_shared_in_ ( nic, ATL2_LINK_OPTS_IN_OFF, &val, 1 );
+	val = 0x4B00FFE1;
+	atl2_hw_write_shared_in_ ( nic, ATL2_LINK_OPTS_IN_OFF, &val, 1 );
+
+	return atl2_hw_finish_ack_ ( nic, 100000);
+}
+
+int atl2_hw_stop ( struct atl_nic *nic )
+{
+	uint32_t val;
+
+	atl2_hw_read_shared_in_ ( nic, ATL2_LINK_OPTS_IN_OFF, &val, 1 );
+	val = 0;
+	atl2_hw_write_shared_in_ ( nic, ATL2_LINK_OPTS_IN_OFF, &val, 1 );
+
+	return atl2_hw_finish_ack_ ( nic, 100000 );
+}
+
+int atl2_hw_get_link ( struct atl_nic *nic )
+{
+	uint32_t val;
+
+	val = ATL_READ_REG ( ATL2_MIF_SHARED_BUF_OUT + ATL2_LINK_STS_OUT_OFF );
+
+	return ( (val & 0xf) != 0) && ((val & 0xF0) != 0 );
+}
+
+int atl2_hw_get_mac ( struct atl_nic *nic, uint8_t *mac )
+{
+	uint32_t mac_addr[2] = {0};
+
+	atl2_hw_read_shared_in_ ( nic, ATL2_MAC_ADDR_IN_OFF, mac_addr, 2 );
+
+	memcpy ( mac, (uint8_t *)mac_addr, 6 );
+
+	return 0;
+}
+
+struct atl_hw_ops atl2_hw = {
+	.reset = atl2_hw_reset,
+	.start = atl2_hw_start,
+	.stop = atl2_hw_stop,
+	.get_link = atl2_hw_get_link,
+	.get_mac = atl2_hw_get_mac,
+};
\ No newline at end of file
diff --git a/src/drivers/net/marvell/atl2_hw.h b/src/drivers/net/marvell/atl2_hw.h
new file mode 100644
index 00000000..ebd5466e
--- /dev/null
+++ b/src/drivers/net/marvell/atl2_hw.h
@@ -0,0 +1,94 @@
+/*
+ * Copyright(C) 2017-2021 Marvell
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Redistribution and use in source and binary forms, with or without 
+ * modification, are permitted provided that the following conditions are met:
+ 
+ *  1. Redistributions of source code must retain the above copyright notice, 
+ *     this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright notice, 
+ *     this list of conditions and the following disclaimer in the documentation 
+ *     and/or other materials provided with the distribution.
+ *   
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS 
+ *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ *   TO,THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ *   PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR HOLDER OR 
+ *   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ *   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ *   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ *   OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ *   WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ *   OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ *   EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef __ATL2_HW_H
+#define __ATL2_HW_H
+
+FILE_LICENCE ( BSD2 );
+
+#define ATL2_GLB_RST_CTRL2 0x3040
+#define ATL2_HOST_FINISHED_WRITE 0xE00
+#define ATL2_MCP_BUSY_WRITE 0xE04
+#define ATL2_HOST_ITR_REQ 0xF00
+
+
+#define ATL2_RESET_STATUS_REQ_GSR		(1U << 0x0)
+#define ATL2_RESET_STATUS_REQ_HOST_BOOT		(1U << 0x8)
+#define ATL2_RESET_STATUS_REQ_MAC_FAST_BOOT	(1U << 0xA)
+#define ATL2_RESET_STATUS_REQ_PHY_FAST_BOOT	(1U << 0xB)
+
+#define ATL2_RESET_STATUS_HOST_LOAD_COMPLETED	(1U << 0x10)
+#define ATL2_RESET_STATUS_REQUIRE_HOST_LOAD	(1U << 0x11)
+#define ATL2_RESET_STATUS_BC_STARTED		(1U << 0x18)
+#define ATL2_RESET_STATUS_CRASH_DURING_INIT	(1U << 0x1B)
+#define ATL2_RESET_STATUS_BC_FAILED		(1U << 0x1C)
+#define ATL2_RESET_STATUS_FW_FAILED		(1U << 0x1D)
+#define ATL2_RESET_STATUS_FW_SUCCEED		(1U << 0x1F)
+
+#define ATL2_RESET_STATUS_BOOT_FAILED_MASK (ATL2_RESET_STATUS_CRASH_DURING_INIT | ATL2_RESET_STATUS_BC_FAILED | ATL2_RESET_STATUS_FW_FAILED)
+#define ATL2_RESET_STATUS_BOOT_COMPLETED_MASK (ATL2_RESET_STATUS_BOOT_FAILED_MASK | ATL2_RESET_STATUS_FW_SUCCEED)
+
+#define ATL2_FW_HOST_INTERRUPT_REQUEST_READY		0x0001
+#define ATL2_FW_HOST_INTERRUPT_MAC_READY		0x0004
+#define ATL2_FW_HOST_INTERRUPT_DATA_HANDLED		0x0100
+#define ATL2_FW_HOST_INTERRUPT_LINK_UP			0x0200
+#define ATL2_FW_HOST_INTERRUPT_LINK_DOWN		0x0400
+#define ATL2_FW_HOST_INTERRUPT_PHY_FAULT		0x0800
+#define ATL2_FW_HOST_INTERRUPT_MAC_FAULT		0x1000
+#define ATL2_FW_HOST_INTERRUPT_TEMPERATURE_WARNING	0x2000
+#define ATL2_FW_HOST_INTERRUPT_HEARTBEAT		0x4000
+
+#define ATL2_FW_LINK_RATE_INVALID 0
+#define ATL2_FW_LINK_RATE_10M     1
+#define ATL2_FW_LINK_RATE_100M    2
+#define ATL2_FW_LINK_RATE_1G      3
+#define ATL2_FW_LINK_RATE_2G5     4
+#define ATL2_FW_LINK_RATE_5G      5
+#define ATL2_FW_LINK_RATE_10G     6
+
+#define ATL2_HOST_MODE_INVALID      0U
+#define ATL2_HOST_MODE_ACTIVE       1U
+#define ATL2_HOST_MODE_SLEEP_PROXY  2U
+#define ATL2_HOST_MODE_LOW_POWER    3U
+#define ATL2_HOST_MODE_SHUTDOWN     4U
+
+#define ATL2_MIF_SHARED_BUF_IN 0x12000
+#define ATL2_MIF_SHARED_BUF_OUT 0x13000
+
+#define ATL2_MTU_IN_OFF 0x0
+#define ATL2_MAC_ADDR_IN_OFF 0x8
+#define ATL2_LINK_CTRL_IN_OFF 0x10
+#define ATL2_LINK_OPTS_IN_OFF 0x18
+
+#define ATL2_FW_OUT_OFF 0x8
+#define ATL2_LINK_STS_OUT_OFF 0x14
+
+#define ATL2_DELAY_10  10
+#define ATL2_DELAY_100  100
+
+#endif
\ No newline at end of file
diff --git a/src/drivers/net/marvell/atl_hw.c b/src/drivers/net/marvell/atl_hw.c
new file mode 100644
index 00000000..2dddb718
--- /dev/null
+++ b/src/drivers/net/marvell/atl_hw.c
@@ -0,0 +1,321 @@
+/** @file
+ *
+ * Marvell AQtion family network card driver, hardware-specific functions.
+ *
+ * Copyright(C) 2017-2021 Marvell
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ 
+ *  1. Redistributions of source code must retain the above copyright notice, 
+ *     this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright notice, 
+ *     this list of conditions and the following disclaimer in the documentation 
+ *     and/or other materials provided with the distribution.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS 
+ *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ *  TO,THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ *   PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR HOLDER OR
+ *   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ *   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ *   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ *   OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ *   WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ *   OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ *   EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+FILE_LICENCE ( BSD2 );
+
+#include <errno.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <byteswap.h>
+#include <ipxe/pci.h>
+#include "aqc1xx.h"
+#include "atl_hw.h"
+#include <compiler.h>
+
+
+int atl_hw_reset_flb_ ( struct atl_nic *nic )
+{
+	uint32_t val;
+	int k = 0;
+
+	ATL_WRITE_REG ( ATL_GLB_CTRL2_MBOX_ERR_UP_RUN_STALL, ATL_GLB_CTRL2 );
+	mdelay ( ATL_DELAY_50_MNS );
+
+	/* Cleanup SPI */
+	val = ATL_READ_REG ( ATL_GLB_NVR_PROV4 );
+	ATL_WRITE_REG ( val | ATL_GBL_NVR_PROV4_RESET, ATL_GLB_NVR_PROV4 );
+
+	ATL_WRITE_REG( (ATL_READ_REG(ATL_GLB_STD_CTRL) &
+		      ~ATL_GLB_CTRL_RST_DIS) | ATL_GLB_STD_CTRL_RESET, 
+			ATL_GLB_STD_CTRL );
+
+	/* Kickstart MAC */
+	ATL_WRITE_REG ( ATL_GLB_CTRL2_FW_RESET, ATL_GLB_CTRL2 );
+	ATL_WRITE_REG ( ATL_MIF_PWR_GATING_EN_CTRL_RESET,
+			ATL_MIF_PWR_GATING_EN_CTRL );
+
+	ATL_WRITE_REG ( ATL_GEN_PROV9_ENABLE, ATL_GEN_PROV9 );
+
+	/* Reset SPI again because of possible interrupted SPI burst */
+	val = ATL_READ_REG ( ATL_GLB_NVR_PROV4 );
+	ATL_WRITE_REG ( val | ATL_GBL_NVR_PROV4_RESET, ATL_GLB_NVR_PROV4 );
+	mdelay ( ATL_DELAY_10_MNS );
+	/* Clear SPI reset state */
+	ATL_WRITE_REG ( val & ~ATL_GBL_NVR_PROV4_RESET, ATL_GLB_NVR_PROV4 );
+
+	/* MAC Kickstart */
+	ATL_WRITE_REG ( ATL_GLB_CTRL2_MAC_KICK_START, ATL_GLB_CTRL2 );
+
+	for (k = 0; k < 1000; k++) {
+		uint32_t flb_status = ATL_READ_REG ( ATL_MPI_DAISY_CHAIN_STS );
+
+		flb_status = flb_status & FLB_LOAD_STS;
+		if ( flb_status )
+			break;
+		mdelay ( ATL_DELAY_10_MNS );
+	}
+	if ( k == 1000 ) {
+		DBGC (nic, "MAC kickstart failed\n" );
+		return -EIO;
+	}
+
+	/* FW reset */
+	ATL_WRITE_REG ( ATL_GLB_CTRL2_FW_RESET, ATL_GLB_CTRL2 );
+	mdelay ( ATL_DELAY_50_MNS );
+
+	ATL_WRITE_REG ( ATL_GBL_MCP_SEM1_RELEASE, ATL_GLB_MCP_SEM1 );
+
+	/* Global software reset*/
+	ATL_WRITE_REG ( ATL_READ_REG ( ATL_RX_CTRL ) & 
+			~ATL_RX_CTRL_RST_DIS, ATL_RX_CTRL );
+	ATL_WRITE_REG ( ATL_READ_REG ( ATL_TX_CTRL ) &
+			~ATL_TX_CTRL_RST_DIS, ATL_TX_CTRL );
+
+	ATL_WRITE_REG ( ATL_READ_REG ( ATL_MAC_PHY_CTRL ) &
+			~ATL_MAC_PHY_CTRL_RST_DIS, ATL_MAC_PHY_CTRL );
+
+	ATL_WRITE_REG ( ( ATL_READ_REG ( ATL_GLB_STD_CTRL ) &
+			~ATL_GLB_CTRL_RST_DIS) | ATL_GLB_STD_CTRL_RESET,
+			ATL_GLB_STD_CTRL );
+
+	for (k = 0; k < 1000; k++) {
+		u32 fw_state = ATL_READ_REG ( ATL_FW_VER );
+
+		if ( fw_state )
+			break;
+		mdelay ( ATL_DELAY_10_MNS );
+	}
+	if ( k == 1000 ) {
+		DBGC ( nic, "FW kickstart failed\n" );
+		return -EIO;
+	}
+	/* Old FW requires fixed delay after init */
+	mdelay ( ATL_DELAY_15_MNS );
+
+	return 0;
+}
+
+int atl_hw_reset_rbl_ ( struct atl_nic *nic )
+{
+	uint32_t val, rbl_status;
+	int k;
+
+	ATL_WRITE_REG ( ATL_GLB_CTRL2_MBOX_ERR_UP_RUN_STALL, ATL_GLB_CTRL2 );
+	ATL_WRITE_REG ( ATL_GBL_MCP_SEM1_RELEASE, ATL_GLB_MCP_SEM1 );
+	ATL_WRITE_REG ( ATL_MIF_PWR_GATING_EN_CTRL_RESET, 
+			ATL_MIF_PWR_GATING_EN_CTRL );
+
+	/* Alter RBL status */
+	ATL_WRITE_REG ( POISON_SIGN, ATL_MPI_BOOT_EXIT_CODE );
+
+	/* Cleanup SPI */
+	val = ATL_READ_REG ( ATL_GLB_NVR_PROV4 );
+	ATL_WRITE_REG ( val | ATL_GBL_NVR_PROV4_RESET, ATL_GLB_NVR_PROV4 );
+
+	/* Global software reset*/
+	ATL_WRITE_REG ( ATL_READ_REG(ATL_RX_CTRL) & ~ATL_RX_CTRL_RST_DIS,
+			ATL_RX_CTRL );
+	ATL_WRITE_REG ( ATL_READ_REG(ATL_TX_CTRL) & ~ATL_TX_CTRL_RST_DIS,
+			ATL_TX_CTRL );
+	ATL_WRITE_REG ( ATL_READ_REG(ATL_MAC_PHY_CTRL) &
+			~ATL_MAC_PHY_CTRL_RST_DIS, ATL_MAC_PHY_CTRL );
+
+	ATL_WRITE_REG ( (ATL_READ_REG(ATL_GLB_STD_CTRL) &
+			~ATL_GLB_CTRL_RST_DIS) | ATL_GLB_STD_CTRL_RESET,
+			ATL_GLB_STD_CTRL );
+
+	ATL_WRITE_REG ( ATL_GLB_CTRL2_MBOX_ERR_UP_RUN_NORMAL, ATL_GLB_CTRL2 );
+
+	/* Wait for RBL boot */
+	for ( k = 0; k < 1000; k++ ) {
+		rbl_status = ATL_READ_REG ( ATL_MPI_BOOT_EXIT_CODE ) & 0xFFFF;
+		if ( rbl_status && rbl_status != POISON_SIGN )
+			break;
+		mdelay ( ATL_DELAY_10_MNS );
+	}
+	if ( !rbl_status || rbl_status == POISON_SIGN ) {
+		DBGC ( nic, "RBL Restart failed\n" );
+		return -EIO;
+	}
+
+	if ( rbl_status == FW_NOT_SUPPORT )
+		return -ENOTSUP;
+
+	for ( k = 0; k < 1000; k++ ) {
+		u32 fw_state = ATL_READ_REG ( ATL_FW_VER );
+
+		if ( fw_state )
+			break;
+		mdelay ( ATL_DELAY_10_MNS );
+	}
+	if ( k == 1000 ) {
+		DBGC ( nic, "FW kickstart failed\n" );
+		return -EIO;
+	}
+	/* Old FW requires fixed delay after init */
+	mdelay ( ATL_DELAY_15_MNS );
+
+	return 0;
+}
+
+int atl_hw_reset ( struct atl_nic *nic )
+{
+	uint32_t boot_exit_code = 0;
+	uint32_t k;
+	int rbl_enabled;
+	uint32_t fw_ver;
+	uint32_t sem_timeout;
+
+	for ( k = 0; k < 1000; ++k ) {
+		uint32_t flb_status = ATL_READ_REG ( ATL_MPI_DAISY_CHAIN_STS );
+		boot_exit_code = ATL_READ_REG ( ATL_MPI_BOOT_EXIT_CODE );
+		if ( flb_status != ATL_MPI_DAISY_CHAIN_STS_ERROR_STATUS ||
+			boot_exit_code != 0 )
+			break;
+	}
+
+	if ( k == 1000 ) {
+		DBGC ( nic, "Neither RBL nor FLB firmware started\n" );
+		return -ENOTSUP;
+	}
+
+	rbl_enabled = (boot_exit_code != 0);
+
+	fw_ver = ATL_READ_REG ( ATL_FW_VER );
+	if ( ((fw_ver >> 24) & 0xFF) >= 4 ) {
+		sem_timeout = ATL_READ_REG ( ATL_SEM_TIMEOUT );
+		if ( sem_timeout > ATL_SEM_MAX_TIMEOUT )
+			sem_timeout = ATL_SEM_MAX_TIMEOUT;
+
+		for ( k = 0; k < sem_timeout; ++k ) {
+			if ( ATL_READ_REG ( ATL_GLB_MCP_SEM4) )
+				break;
+
+			mdelay (ATL_DELAY_1_MNS);
+		}
+		for ( k = 0; k < sem_timeout; ++k ) {
+			if (ATL_READ_REG ( ATL_GLB_MCP_SEM5) )
+				break;
+
+			mdelay ( ATL_DELAY_1_MNS );
+		}
+	}
+
+
+	if ( rbl_enabled )
+		return atl_hw_reset_rbl_ ( nic );
+	else
+		return atl_hw_reset_flb_ ( nic );
+}
+
+int atl_hw_start ( struct atl_nic *nic )
+{
+	ATL_WRITE_REG ( ATL_LINK_ADV_AUTONEG, ATL_LINK_ADV );
+	return 0;
+}
+
+int atl_hw_stop ( struct atl_nic *nic )
+{
+	ATL_WRITE_REG ( ATL_SHUT_LINK, ATL_LINK_ADV );
+	return 0;
+}
+
+int atl_hw_get_link ( struct atl_nic *nic )
+{
+	return ( ATL_READ_REG ( ATL_LINK_ST) & ATL_LINK_ADV_AUTONEG ) != 0;
+}
+
+int atl_hw_read_mem ( struct atl_nic *nic, uint32_t addr, uint32_t *buffer,
+		    uint32_t size )
+{
+	uint32_t i;
+
+	for ( i = 0; i < 100; ++i ) {
+		if ( ATL_READ_REG( ATL_SEM_RAM) )
+			break;
+		mdelay ( ATL_DELAY_1_MNS );
+	}
+	if ( i == 100 ) {
+		DBGC (nic, "Semaphore Register not set\n" );
+		return -EIO;
+	}
+
+	ATL_WRITE_REG ( addr, ATL_MBOX_CTRL3 );
+
+	for ( i = 0; i < size; ++i, addr += 4 ) {
+		uint32_t j;
+
+		ATL_WRITE_REG ( ATL_MBOX_CTRL1_START_MBOX_OPT, ATL_MBOX_CTRL1 );
+		for ( j = 0; j < 10000; ++j ) {
+			if ( ATL_READ_REG (ATL_MBOX_CTRL3 ) != addr )
+				break;
+			udelay ( ATL_DELAY_10_MNS );
+		}
+		if ( j == 10000 ) {
+			DBGC (nic, "Reading from CTRL3 Register Failed\n" );
+			return -EIO;
+		}
+
+		buffer[i] = ATL_READ_REG ( ATL_MBOX_CTRL5 );
+	}
+
+	ATL_WRITE_REG( ATL_SEM_RAM_RESET, ATL_SEM_RAM );
+
+	return 0;
+}
+
+int atl_hw_get_mac ( struct atl_nic *nic, uint8_t *mac )
+{
+	uint32_t mac_addr[2] = {0};
+	int err = 0;
+	uint32_t efuse_addr = ATL_READ_REG ( ATL_GLB_MCP_SP26 );
+
+	if ( efuse_addr != 0) {
+		uint32_t mac_efuse_addr = efuse_addr + 40 * sizeof(uint32_t);
+		err = atl_hw_read_mem ( nic, mac_efuse_addr, mac_addr, 2 );
+		if ( err != 0 )
+			return err;
+
+		mac_addr[0] = cpu_to_be32 ( mac_addr[0] );
+		mac_addr[1] = cpu_to_be32 ( mac_addr[1] );
+
+		memcpy ( mac, (uint8_t *)mac_addr, ATL_MAC_ADDRESS_SIZE );
+	}
+	return 0;
+}
+
+struct atl_hw_ops atl_hw = {
+	.reset = atl_hw_reset,
+	.start = atl_hw_start,
+	.stop = atl_hw_stop,
+	.get_link = atl_hw_get_link,
+	.get_mac = atl_hw_get_mac,
+};
\ No newline at end of file
diff --git a/src/drivers/net/marvell/atl_hw.h b/src/drivers/net/marvell/atl_hw.h
new file mode 100644
index 00000000..0a20fbfc
--- /dev/null
+++ b/src/drivers/net/marvell/atl_hw.h
@@ -0,0 +1,83 @@
+/*
+ * Copyright(C) 2017-2021 Marvell
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Redistribution and use in source and binary forms, with or without 
+ * modification, are permitted provided that the following conditions are met:
+ 
+ *  1. Redistributions of source code must retain the above copyright notice, 
+ *     this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright notice, 
+ *     this list of conditions and the following disclaimer in the documentation 
+ *     and/or other materials provided with the distribution.
+ *   
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS 
+ *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ *   TO,THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ *   PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR HOLDER OR 
+ *   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ *   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ *   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ *   OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ *   WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ *   OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ *   EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#ifndef __ATL_HW_H
+#define __ATL_HW_H
+
+FILE_LICENCE ( BSD2 );
+
+#define ATL_GLB_STD_CTRL 0x0
+#define ATL_GLB_CTRL_RST_DIS 0x4000
+#define ATL_FW_VER 0x18
+
+#define ATL_MPI_DAISY_CHAIN_STS 0x704
+#define ATL_MPI_RX_DAISY_CHAIN_DATA 0x04000000
+#define ATL_MPI_RX_DAISY_CHAIN_SOF  0x02000000
+#define FLB_LOAD_STS 0x10
+
+#define ATL_MPI_BOOT_EXIT_CODE 0x388
+
+#define ATL_SEM_TIMEOUT 0x348
+#define ATL_SEM_MAX_TIMEOUT 3000
+
+#define ATL_GLB_CTRL2 0x404
+#define ATL_GLB_MCP_SEM1 0x3A0
+#define ATL_GBL_MCP_SEM1_RELEASE 0x1
+
+#define ATL_GLB_MCP_SEM4 0x3AC
+#define ATL_GLB_MCP_SEM5 0x3B0
+#define ATL_GLB_MCP_SP26 0x364
+#define ATL_MIF_PWR_GATING_EN_CTRL 0x32A8
+
+#define ATL_GLB_NVR_PROV4       0x53C
+#define ATL_GBL_NVR_PROV4_RESET 0x10
+
+
+#define ATL_GEN_PROV9 0x520
+
+#define ATL_MAC_PHY_CTRL 0x00004000U
+#define ATL_MAC_PHY_CTRL_RST_DIS 0x20000000U
+
+#define ATL_MIF_PWR_GATING_EN_CTRL_RESET      0x0
+#define ATL_GEN_PROV9_ENABLE                  0x1
+#define ATL_GLB_CTRL2_MAC_KICK_START          0x180e0
+#define ATL_GLB_CTRL2_FW_RESET                0x80e0
+#define ATL_GLB_CTRL2_MBOX_ERR_UP_RUN_STALL   0x40e1
+#define ATL_GLB_CTRL2_MBOX_ERR_UP_RUN_NORMAL  0x40e0
+#define ATL_GLB_STD_CTRL_RESET                0x8000
+#define ATL_MPI_DAISY_CHAIN_STS_ERROR_STATUS  0x06000000
+
+#define ATL_DELAY_1_MNS   1
+#define ATL_DELAY_10_MNS  10
+#define ATL_DELAY_15_MNS  15
+#define ATL_DELAY_50_MNS  50
+
+#define ATL_MAC_ADDRESS_SIZE 6
+#define POISON_SIGN 0xDEAD
+#define FW_NOT_SUPPORT 0xF1A7
+
+#endif
diff --git a/src/drivers/net/ns8390.c b/src/drivers/net/ns8390.c
index 0ffc6216..8e8d8500 100644
--- a/src/drivers/net/ns8390.c
+++ b/src/drivers/net/ns8390.c
@@ -1006,17 +1006,17 @@ ISA_ROM("ne","NE1000/2000 and clones");
 #ifdef	INCLUDE_NS8390
 static struct pci_device_id nepci_nics[] = {
 /* A few NE2000 PCI clones, list not exhaustive */
-PCI_ROM(0x10ec, 0x8029, "rtl8029",      "Realtek 8029", 0),
-PCI_ROM(0x1186, 0x0300, "dlink-528",    "D-Link DE-528", 0),
 PCI_ROM(0x1050, 0x0940, "winbond940",   "Winbond NE2000-PCI", 0),		/* Winbond 86C940 / 89C940 */
 PCI_ROM(0x1050, 0x5a5a, "winbond940f",  "Winbond W89c940F", 0),		/* Winbond 89C940F */
+PCI_ROM(0x10bd, 0x0e34, "surecom-ne34", "Surecom NE34", 0),
+PCI_ROM(0x10ec, 0x8029, "rtl8029",      "Realtek 8029", 0),
+PCI_ROM(0x1106, 0x0926, "via86c926",    "Via 86c926", 0),
+PCI_ROM(0x1186, 0x0300, "dlink-528",    "D-Link DE-528", 0),
 PCI_ROM(0x11f6, 0x1401, "compexrl2000", "Compex ReadyLink 2000", 0),
-PCI_ROM(0x8e2e, 0x3000, "ktiet32p2",    "KTI ET32P2", 0),
-PCI_ROM(0x4a14, 0x5000, "nv5000sc",     "NetVin NV5000SC", 0),
 PCI_ROM(0x12c3, 0x0058, "holtek80232",  "Holtek HT80232", 0),
 PCI_ROM(0x12c3, 0x5598, "holtek80229",  "Holtek HT80229", 0),
-PCI_ROM(0x10bd, 0x0e34, "surecom-ne34", "Surecom NE34", 0),
-PCI_ROM(0x1106, 0x0926, "via86c926",    "Via 86c926", 0),
+PCI_ROM(0x4a14, 0x5000, "nv5000sc",     "NetVin NV5000SC", 0),
+PCI_ROM(0x8e2e, 0x3000, "ktiet32p2",    "KTI ET32P2", 0),
 };
 
 PCI_DRIVER ( nepci_driver, nepci_nics, PCI_NO_CLASS );
diff --git a/src/drivers/net/pcnet32.c b/src/drivers/net/pcnet32.c
index 7da884e5..a9286d6a 100644
--- a/src/drivers/net/pcnet32.c
+++ b/src/drivers/net/pcnet32.c
@@ -1149,8 +1149,8 @@ pcnet32_remove ( struct pci_device *pdev )
 
 static struct pci_device_id pcnet32_nics[] = {
 	PCI_ROM(0x1022, 0x2000, "pcnet32", "AMD PCnet/PCI", 0),
-	PCI_ROM(0x1022, 0x2625, "pcnetfastiii", "AMD PCNet FAST III", 0),
 	PCI_ROM(0x1022, 0x2001, "amdhomepna", "AMD PCnet/HomePNA", 0),
+	PCI_ROM(0x1022, 0x2625, "pcnetfastiii", "AMD PCNet FAST III", 0),
 };
 
 struct pci_driver pcnet32_driver __pci_driver = {
diff --git a/src/drivers/net/prism2_plx.c b/src/drivers/net/prism2_plx.c
index a73b0e08..770cf328 100644
--- a/src/drivers/net/prism2_plx.c
+++ b/src/drivers/net/prism2_plx.c
@@ -104,9 +104,10 @@ static void prism2_plx_disable ( struct nic *nic ) {
 }
 
 static struct pci_device_id prism2_plx_nics[] = {
-PCI_ROM(0x1385, 0x4100, "ma301",         "Netgear MA301", 0),
 PCI_ROM(0x10b7, 0x7770, "3c-airconnect", "3Com AirConnect", 0),
 PCI_ROM(0x111a, 0x1023, "ss1023",        "Siemens SpeedStream SS1023", 0),
+PCI_ROM(0x126c, 0x8030, "emobility",     "Nortel emobility", 0),
+PCI_ROM(0x1385, 0x4100, "ma301",         "Netgear MA301", 0),
 PCI_ROM(0x15e8, 0x0130, "correga",       "Correga", 0),
 PCI_ROM(0x1638, 0x1100, "smc2602w",      "SMC EZConnect SMC2602W", 0),	/* or Eumitcom PCI WL11000, Addtron AWA-100 */
 PCI_ROM(0x16ab, 0x1100, "gl24110p",      "Global Sun Tech GL24110P", 0),
@@ -114,7 +115,6 @@ PCI_ROM(0x16ab, 0x1101, "16ab-1101",     "Unknown", 0),
 PCI_ROM(0x16ab, 0x1102, "wdt11",         "Linksys WDT11", 0),
 PCI_ROM(0x16ec, 0x3685, "usr2415",       "USR 2415", 0),
 PCI_ROM(0xec80, 0xec00, "f5d6000",       "Belkin F5D6000", 0),
-PCI_ROM(0x126c, 0x8030, "emobility",     "Nortel emobility", 0),
 };
 
 PCI_DRIVER ( prism2_plx_driver, prism2_plx_nics, PCI_NO_CLASS );
diff --git a/src/drivers/net/rhine.c b/src/drivers/net/rhine.c
index f4d3a258..fa0876ad 100644
--- a/src/drivers/net/rhine.c
+++ b/src/drivers/net/rhine.c
@@ -775,10 +775,10 @@ static void rhine_remove ( struct pci_device *pci ) {
 
 /** Rhine PCI device IDs */
 static struct pci_device_id rhine_nics[] = {
-	PCI_ROM ( 0x1106, 0x3065, "dlink-530tx", "VIA VT6102", 0 ),
-	PCI_ROM ( 0x1106, 0x3106, "vt6105", "VIA VT6105", 0 ),
 	PCI_ROM ( 0x1106, 0x3043, "dlink-530tx-old", "VIA VT3043", 0 ),
 	PCI_ROM ( 0x1106, 0x3053, "vt6105m", "VIA VT6105M", 0 ),
+	PCI_ROM ( 0x1106, 0x3065, "dlink-530tx", "VIA VT6102", 0 ),
+	PCI_ROM ( 0x1106, 0x3106, "vt6105", "VIA VT6105", 0 ),
 	PCI_ROM ( 0x1106, 0x6100, "via-rhine-old", "VIA 86C100A", 0 )
 };
 
diff --git a/src/drivers/net/rtl818x/rtl8180.c b/src/drivers/net/rtl818x/rtl8180.c
index 5f97480f..b3f68541 100644
--- a/src/drivers/net/rtl818x/rtl8180.c
+++ b/src/drivers/net/rtl818x/rtl8180.c
@@ -7,9 +7,9 @@ FILE_LICENCE(GPL2_OR_LATER);
 
 static struct pci_device_id rtl8180_nics[] = {
 	PCI_ROM(0x10ec, 0x8180, "rtl8180", "Realtek 8180", 0),
+	PCI_ROM(0x1186, 0x3300, "dwl510",  "D-Link DWL-510", 0),
 	PCI_ROM(0x1799, 0x6001, "f5d6001", "Belkin F5D6001", 0),
 	PCI_ROM(0x1799, 0x6020, "f5d6020", "Belkin F5D6020", 0),
-	PCI_ROM(0x1186, 0x3300, "dwl510",  "D-Link DWL-510", 0),
 };
 
 struct pci_driver rtl8180_driver __pci_driver = {
diff --git a/src/drivers/net/sky2.c b/src/drivers/net/sky2.c
index 26396585..4f8ec3e4 100644
--- a/src/drivers/net/sky2.c
+++ b/src/drivers/net/sky2.c
@@ -81,8 +81,8 @@ FILE_LICENCE ( GPL2_ONLY );
 static struct pci_device_id sky2_id_table[] = {
 	PCI_ROM(0x1148, 0x9000, "sk9sxx", "Syskonnect SK-9Sxx", 0),
 	PCI_ROM(0x1148, 0x9e00, "sk9exx", "Syskonnect SK-9Exx", 0),
-	PCI_ROM(0x1186, 0x4b00, "dge560t", "D-Link DGE-560T", 0),
 	PCI_ROM(0x1186, 0x4001, "dge550sx", "D-Link DGE-550SX", 0),
+	PCI_ROM(0x1186, 0x4b00, "dge560t", "D-Link DGE-560T", 0),
 	PCI_ROM(0x1186, 0x4b02, "dge560sx", "D-Link DGE-560SX", 0),
 	PCI_ROM(0x1186, 0x4b03, "dge550t", "D-Link DGE-550T", 0),
 	PCI_ROM(0x11ab, 0x4340, "m88e8021", "Marvell 88E8021", 0),
diff --git a/src/drivers/net/sundance.c b/src/drivers/net/sundance.c
index 9127fa2c..8eb09b98 100644
--- a/src/drivers/net/sundance.c
+++ b/src/drivers/net/sundance.c
@@ -880,9 +880,9 @@ static void set_rx_mode(struct nic *nic __unused)
 }
 
 static struct pci_device_id sundance_nics[] = {
-	PCI_ROM(0x13f0, 0x0201, "sundance", "ST201 Sundance 'Alta' based Adaptor", 0),
 	PCI_ROM(0x1186, 0x1002, "dfe530txs", "D-Link DFE530TXS (Sundance ST201 Alta)", 0),
 	PCI_ROM(0x13f0, 0x0200, "ip100a", "IC+ IP100A", 0),
+	PCI_ROM(0x13f0, 0x0201, "sundance", "ST201 Sundance 'Alta' based Adaptor", 0),
 };
 
 PCI_DRIVER ( sundance_driver, sundance_nics, PCI_NO_CLASS );
diff --git a/src/drivers/net/tg3/tg3.c b/src/drivers/net/tg3/tg3.c
index 559c2d63..05af22d6 100644
--- a/src/drivers/net/tg3/tg3.c
+++ b/src/drivers/net/tg3/tg3.c
@@ -856,88 +856,88 @@ static void tg3_remove_one(struct pci_device *pci)
 }
 
 static struct pci_device_id tg3_nics[] = {
+	PCI_ROM(0x106b, 0x1645, "106b-1645", "106b-1645", 0),
+	PCI_ROM(0x1148, 0x4400, "1148-4400", "1148-4400", 0),
+	PCI_ROM(0x1148, 0x4500, "1148-4500", "1148-4500", 0),
+	PCI_ROM(0x14e4, 0x1600, "14e4-1600", "14e4-1600", 0),
+	PCI_ROM(0x14e4, 0x1601, "14e4-1601", "14e4-1601", 0),
 	PCI_ROM(0x14e4, 0x1644, "14e4-1644", "14e4-1644", 0),
 	PCI_ROM(0x14e4, 0x1645, "14e4-1645", "14e4-1645", 0),
 	PCI_ROM(0x14e4, 0x1646, "14e4-1646", "14e4-1646", 0),
 	PCI_ROM(0x14e4, 0x1647, "14e4-1647", "14e4-1647", 0),
 	PCI_ROM(0x14e4, 0x1648, "14e4-1648", "14e4-1648", 0),
+	PCI_ROM(0x14e4, 0x1649, "14e4-1649", "14e4-1649", 0),
 	PCI_ROM(0x14e4, 0x164d, "14e4-164d", "14e4-164d", 0),
 	PCI_ROM(0x14e4, 0x1653, "14e4-1653", "14e4-1653", 0),
 	PCI_ROM(0x14e4, 0x1654, "14e4-1654", "14e4-1654", 0),
+	PCI_ROM(0x14e4, 0x1655, "14e4-1655", "14e4-1655", 0),
+	PCI_ROM(0x14e4, 0x1656, "14e4-1656", "14e4-1656", 0),
+	PCI_ROM(0x14e4, 0x1657, "14e4-1657", "14e4-1657", 0),
+	PCI_ROM(0x14e4, 0x1659, "14e4-1659", "14e4-1659", 0),
+	PCI_ROM(0x14e4, 0x165a, "14e4-165a", "14e4-165a", 0),
+	PCI_ROM(0x14e4, 0x165b, "14e4-165b", "14e4-165b", 0),
 	PCI_ROM(0x14e4, 0x165d, "14e4-165d", "14e4-165d", 0),
 	PCI_ROM(0x14e4, 0x165e, "14e4-165e", "14e4-165e", 0),
-	PCI_ROM(0x14e4, 0x16a6, "14e4-16a6", "14e4-16a6", 0),
-	PCI_ROM(0x14e4, 0x16a7, "14e4-16a7", "14e4-16a7", 0),
-	PCI_ROM(0x14e4, 0x16a8, "14e4-16a8", "14e4-16a8", 0),
-	PCI_ROM(0x14e4, 0x16c6, "14e4-16c6", "14e4-16c6", 0),
-	PCI_ROM(0x14e4, 0x16c7, "14e4-16c7", "14e4-16c7", 0),
-	PCI_ROM(0x14e4, 0x1696, "14e4-1696", "14e4-1696", 0),
-	PCI_ROM(0x14e4, 0x169c, "14e4-169c", "14e4-169c", 0),
-	PCI_ROM(0x14e4, 0x169d, "14e4-169d", "14e4-169d", 0),
-	PCI_ROM(0x14e4, 0x170d, "14e4-170d", "14e4-170d", 0),
-	PCI_ROM(0x14e4, 0x170e, "14e4-170e", "14e4-170e", 0),
-	PCI_ROM(0x14e4, 0x1649, "14e4-1649", "14e4-1649", 0),
+	PCI_ROM(0x14e4, 0x165f, "14e4-165f", "14e4-165f", 0),
+	PCI_ROM(0x14e4, 0x1668, "14e4-1668", "14e4-1668", 0),
+	PCI_ROM(0x14e4, 0x1669, "14e4-1669", "14e4-1669", 0),
+	PCI_ROM(0x14e4, 0x166a, "14e4-166a", "14e4-166a", 0),
+	PCI_ROM(0x14e4, 0x166b, "14e4-166b", "14e4-166b", 0),
 	PCI_ROM(0x14e4, 0x166e, "14e4-166e", "14e4-166e", 0),
-	PCI_ROM(0x14e4, 0x1659, "14e4-1659", "14e4-1659", 0),
-	PCI_ROM(0x14e4, 0x165a, "14e4-165a", "14e4-165a", 0),
-	PCI_ROM(0x14e4, 0x1677, "14e4-1677", "14e4-1677", 0),
-	PCI_ROM(0x14e4, 0x167d, "14e4-167d", "14e4-167d", 0),
-	PCI_ROM(0x14e4, 0x167e, "14e4-167e", "14e4-167e", 0),
-	PCI_ROM(0x14e4, 0x1600, "14e4-1600", "14e4-1600", 0),
-	PCI_ROM(0x14e4, 0x1601, "14e4-1601", "14e4-1601", 0),
-	PCI_ROM(0x14e4, 0x16f7, "14e4-16f7", "14e4-16f7", 0),
-	PCI_ROM(0x14e4, 0x16fd, "14e4-16fd", "14e4-16fd", 0),
-	PCI_ROM(0x14e4, 0x16fe, "14e4-16fe", "14e4-16fe", 0),
-	PCI_ROM(0x14e4, 0x167a, "14e4-167a", "14e4-167a", 0),
 	PCI_ROM(0x14e4, 0x1672, "14e4-1672", "14e4-1672", 0),
-	PCI_ROM(0x14e4, 0x167b, "14e4-167b", "14e4-167b", 0),
 	PCI_ROM(0x14e4, 0x1673, "14e4-1673", "14e4-1673", 0),
 	PCI_ROM(0x14e4, 0x1674, "14e4-1674", "14e4-1674", 0),
-	PCI_ROM(0x14e4, 0x169a, "14e4-169a", "14e4-169a", 0),
-	PCI_ROM(0x14e4, 0x169b, "14e4-169b", "14e4-169b", 0),
-	PCI_ROM(0x14e4, 0x1693, "14e4-1693", "14e4-1693", 0),
-	PCI_ROM(0x14e4, 0x167f, "14e4-167f", "14e4-167f", 0),
-	PCI_ROM(0x14e4, 0x1668, "14e4-1668", "14e4-1668", 0),
-	PCI_ROM(0x14e4, 0x1669, "14e4-1669", "14e4-1669", 0),
+	PCI_ROM(0x14e4, 0x1677, "14e4-1677", "14e4-1677", 0),
 	PCI_ROM(0x14e4, 0x1678, "14e4-1678", "14e4-1678", 0),
 	PCI_ROM(0x14e4, 0x1679, "14e4-1679", "14e4-1679", 0),
-	PCI_ROM(0x14e4, 0x166a, "14e4-166a", "14e4-166a", 0),
-	PCI_ROM(0x14e4, 0x166b, "14e4-166b", "14e4-166b", 0),
-	PCI_ROM(0x14e4, 0x16dd, "14e4-16dd", "14e4-16dd", 0),
-	PCI_ROM(0x14e4, 0x1712, "14e4-1712", "14e4-1712", 0),
-	PCI_ROM(0x14e4, 0x1713, "14e4-1713", "14e4-1713", 0),
-	PCI_ROM(0x14e4, 0x1698, "14e4-1698", "14e4-1698", 0),
-	PCI_ROM(0x14e4, 0x1684, "14e4-1684", "14e4-1684", 0),
-	PCI_ROM(0x14e4, 0x165b, "14e4-165b", "14e4-165b", 0),
+	PCI_ROM(0x14e4, 0x167a, "14e4-167a", "14e4-167a", 0),
+	PCI_ROM(0x14e4, 0x167b, "14e4-167b", "14e4-167b", 0),
+	PCI_ROM(0x14e4, 0x167d, "14e4-167d", "14e4-167d", 0),
+	PCI_ROM(0x14e4, 0x167e, "14e4-167e", "14e4-167e", 0),
+	PCI_ROM(0x14e4, 0x167f, "14e4-167f", "14e4-167f", 0),
+	PCI_ROM(0x14e4, 0x1680, "14e4-1680", "14e4-1680", 0),
 	PCI_ROM(0x14e4, 0x1681, "14e4-1681", "14e4-1681", 0),
 	PCI_ROM(0x14e4, 0x1682, "14e4-1682", "14e4-1682", 0),
-	PCI_ROM(0x14e4, 0x1680, "14e4-1680", "14e4-1680", 0),
+	PCI_ROM(0x14e4, 0x1684, "14e4-1684", "14e4-1684", 0),
+	PCI_ROM(0x14e4, 0x1686, "14e4-1686", "14e4-1686", 0),
 	PCI_ROM(0x14e4, 0x1688, "14e4-1688", "14e4-1688", 0),
 	PCI_ROM(0x14e4, 0x1689, "14e4-1689", "14e4-1689", 0),
-	PCI_ROM(0x14e4, 0x1699, "14e4-1699", "14e4-1699", 0),
-	PCI_ROM(0x14e4, 0x16a0, "14e4-16a0", "14e4-16a0", 0),
-	PCI_ROM(0x14e4, 0x1692, "14e4-1692", "14e4-1692", 0),
 	PCI_ROM(0x14e4, 0x1690, "14e4-1690", "14e4-1690", 0),
-	PCI_ROM(0x14e4, 0x1694, "14e4-1694", "14e4-1694", 0),
 	PCI_ROM(0x14e4, 0x1691, "14e4-1691", "14e4-1691", 0),
-	PCI_ROM(0x14e4, 0x1655, "14e4-1655", "14e4-1655", 0),
-	PCI_ROM(0x14e4, 0x1656, "14e4-1656", "14e4-1656", 0),
-	PCI_ROM(0x14e4, 0x16b1, "14e4-16b1", "14e4-16b1", 0),
-	PCI_ROM(0x14e4, 0x16b5, "14e4-16b5", "14e4-16b5", 0),
+	PCI_ROM(0x14e4, 0x1692, "14e4-1692", "14e4-1692", 0),
+	PCI_ROM(0x14e4, 0x1693, "14e4-1693", "14e4-1693", 0),
+	PCI_ROM(0x14e4, 0x1694, "14e4-1694", "14e4-1694", 0),
+	PCI_ROM(0x14e4, 0x1696, "14e4-1696", "14e4-1696", 0),
+	PCI_ROM(0x14e4, 0x1698, "14e4-1698", "14e4-1698", 0),
+	PCI_ROM(0x14e4, 0x1699, "14e4-1699", "14e4-1699", 0),
+	PCI_ROM(0x14e4, 0x169a, "14e4-169a", "14e4-169a", 0),
+	PCI_ROM(0x14e4, 0x169b, "14e4-169b", "14e4-169b", 0),
+	PCI_ROM(0x14e4, 0x169c, "14e4-169c", "14e4-169c", 0),
+	PCI_ROM(0x14e4, 0x169d, "14e4-169d", "14e4-169d", 0),
+	PCI_ROM(0x14e4, 0x16a0, "14e4-16a0", "14e4-16a0", 0),
+	PCI_ROM(0x14e4, 0x16a6, "14e4-16a6", "14e4-16a6", 0),
+	PCI_ROM(0x14e4, 0x16a7, "14e4-16a7", "14e4-16a7", 0),
+	PCI_ROM(0x14e4, 0x16a8, "14e4-16a8", "14e4-16a8", 0),
 	PCI_ROM(0x14e4, 0x16b0, "14e4-16b0", "14e4-16b0", 0),
-	PCI_ROM(0x14e4, 0x16b4, "14e4-16b4", "14e4-16b4", 0),
+	PCI_ROM(0x14e4, 0x16b1, "14e4-16b1", "14e4-16b1", 0),
 	PCI_ROM(0x14e4, 0x16b2, "14e4-16b2", "14e4-16b2", 0),
+	PCI_ROM(0x14e4, 0x16b4, "14e4-16b4", "14e4-16b4", 0),
+	PCI_ROM(0x14e4, 0x16b5, "14e4-16b5", "14e4-16b5", 0),
 	PCI_ROM(0x14e4, 0x16b6, "14e4-16b6", "14e4-16b6", 0),
-	PCI_ROM(0x14e4, 0x1657, "14e4-1657", "14e4-1657", 0),
-	PCI_ROM(0x14e4, 0x165f, "14e4-165f", "14e4-165f", 0),
-	PCI_ROM(0x14e4, 0x1686, "14e4-1686", "14e4-1686", 0),
-	PCI_ROM(0x1148, 0x4400, "1148-4400", "1148-4400", 0),
-	PCI_ROM(0x1148, 0x4500, "1148-4500", "1148-4500", 0),
+	PCI_ROM(0x14e4, 0x16c6, "14e4-16c6", "14e4-16c6", 0),
+	PCI_ROM(0x14e4, 0x16c7, "14e4-16c7", "14e4-16c7", 0),
+	PCI_ROM(0x14e4, 0x16dd, "14e4-16dd", "14e4-16dd", 0),
+	PCI_ROM(0x14e4, 0x16f7, "14e4-16f7", "14e4-16f7", 0),
+	PCI_ROM(0x14e4, 0x16fd, "14e4-16fd", "14e4-16fd", 0),
+	PCI_ROM(0x14e4, 0x16fe, "14e4-16fe", "14e4-16fe", 0),
+	PCI_ROM(0x14e4, 0x170d, "14e4-170d", "14e4-170d", 0),
+	PCI_ROM(0x14e4, 0x170e, "14e4-170e", "14e4-170e", 0),
+	PCI_ROM(0x14e4, 0x1712, "14e4-1712", "14e4-1712", 0),
+	PCI_ROM(0x14e4, 0x1713, "14e4-1713", "14e4-1713", 0),
 	PCI_ROM(0x173b, 0x03e8, "173b-03e8", "173b-03e8", 0),
 	PCI_ROM(0x173b, 0x03e9, "173b-03e9", "173b-03e9", 0),
-	PCI_ROM(0x173b, 0x03eb, "173b-03eb", "173b-03eb", 0),
 	PCI_ROM(0x173b, 0x03ea, "173b-03ea", "173b-03ea", 0),
-	PCI_ROM(0x106b, 0x1645, "106b-1645", "106b-1645", 0),
+	PCI_ROM(0x173b, 0x03eb, "173b-03eb", "173b-03eb", 0),
 };
 
 struct pci_driver tg3_pci_driver __pci_driver = {
diff --git a/src/drivers/net/tlan.c b/src/drivers/net/tlan.c
index 0e85b35b..93533b43 100644
--- a/src/drivers/net/tlan.c
+++ b/src/drivers/net/tlan.c
@@ -1697,19 +1697,19 @@ void TLan_PhyMonitor(struct net_device *dev)
 #endif				/* MONITOR */
 
 static struct pci_device_id tlan_nics[] = {
-	PCI_ROM(0x0e11, 0xae34, "netel10", "Compaq Netelligent 10 T PCI UTP", 0),
 	PCI_ROM(0x0e11, 0xae32, "netel100","Compaq Netelligent 10/100 TX PCI UTP", 0),
+	PCI_ROM(0x0e11, 0xae34, "netel10", "Compaq Netelligent 10 T PCI UTP", 0),
 	PCI_ROM(0x0e11, 0xae35, "netflex3i", "Compaq Integrated NetFlex-3/P", 0),
-	PCI_ROM(0x0e11, 0xf130, "thunder", "Compaq NetFlex-3/P", 0),
-	PCI_ROM(0x0e11, 0xf150, "netflex3b", "Compaq NetFlex-3/P", 0),
-	PCI_ROM(0x0e11, 0xae43, "netel100pi", "Compaq Netelligent Integrated 10/100 TX UTP", 0),
 	PCI_ROM(0x0e11, 0xae40, "netel100d", "Compaq Netelligent Dual 10/100 TX PCI UTP", 0),
+	PCI_ROM(0x0e11, 0xae43, "netel100pi", "Compaq Netelligent Integrated 10/100 TX UTP", 0),
 	PCI_ROM(0x0e11, 0xb011, "netel100i", "Compaq Netelligent 10/100 TX Embedded UTP", 0),
-	PCI_ROM(0x108d, 0x0013, "oc2183", "Olicom OC-2183/2185", 0),
+	PCI_ROM(0x0e11, 0xb012, "netelligent_10_t2", "Compaq Netelligent 10 T/2 PCI UTP/Coax", 0),
+	PCI_ROM(0x0e11, 0xb030, "netelligent_10_100_ws_5100", "Compaq Netelligent 10/100 TX UTP", 0),
+	PCI_ROM(0x0e11, 0xf130, "thunder", "Compaq NetFlex-3/P", 0),
+	PCI_ROM(0x0e11, 0xf150, "netflex3b", "Compaq NetFlex-3/P", 0),
 	PCI_ROM(0x108d, 0x0012, "oc2325", "Olicom OC-2325", 0),
+	PCI_ROM(0x108d, 0x0013, "oc2183", "Olicom OC-2183/2185", 0),
 	PCI_ROM(0x108d, 0x0014, "oc2326", "Olicom OC-2326", 0),
-	PCI_ROM(0x0e11, 0xb030, "netelligent_10_100_ws_5100", "Compaq Netelligent 10/100 TX UTP", 0),
-	PCI_ROM(0x0e11, 0xb012, "netelligent_10_t2", "Compaq Netelligent 10 T/2 PCI UTP/Coax", 0),
 };
 
 PCI_DRIVER ( tlan_driver, tlan_nics, PCI_NO_CLASS );
diff --git a/src/drivers/net/tulip.c b/src/drivers/net/tulip.c
index e4e6ffa8..fddebfe5 100644
--- a/src/drivers/net/tulip.c
+++ b/src/drivers/net/tulip.c
@@ -1921,31 +1921,30 @@ PCI_ROM(0x1011, 0x0002, "dc21040",     "Digital Tulip", 0),
 PCI_ROM(0x1011, 0x0009, "ds21140",     "Digital Tulip Fast", 0),
 PCI_ROM(0x1011, 0x0014, "dc21041",     "Digital Tulip+", 0),
 PCI_ROM(0x1011, 0x0019, "ds21142",     "Digital Tulip 21142", 0),
+PCI_ROM(0x104a, 0x0981, "tulip-0981",  "Tulip 0x104a 0x0981", 0),
+PCI_ROM(0x104a, 0x2774, "SGThomson-STE10100A", "Tulip 0x104a 0x2774", 0),	/*Modified by Ramesh Chander*/
 PCI_ROM(0x10b7, 0x9300, "3csoho100b-tx","3ComSOHO100B-TX", 0),
 PCI_ROM(0x10b9, 0x5261, "ali1563",     "ALi 1563 integrated ethernet", 0),
 PCI_ROM(0x10d9, 0x0512, "mx98713",     "Macronix MX987x3", 0),
 PCI_ROM(0x10d9, 0x0531, "mx98715",     "Macronix MX987x5", 0),
+PCI_ROM(0x1113, 0x1216, "an983",       "ADMTek AN983 Comet", 0),
 PCI_ROM(0x1113, 0x1217, "mxic-98715",  "Macronix MX987x5", 0),
-PCI_ROM(0x11ad, 0xc115, "lc82c115",    "LinkSys LNE100TX", 0),
+PCI_ROM(0x1113, 0x9511, "tulip-9511",  "Tulip 0x1113 0x9511", 0),
+PCI_ROM(0x115d, 0x0003, "xircomtulip", "Xircom Tulip", 0),
+PCI_ROM(0x1186, 0x1561, "tulip-1561",  "Tulip 0x1186 0x1561", 0),
 PCI_ROM(0x11ad, 0x0002, "82c168",      "Netgear FA310TX", 0),
+PCI_ROM(0x11ad, 0xc115, "lc82c115",    "LinkSys LNE100TX", 0),
+PCI_ROM(0x11f6, 0x9881, "rl100tx",     "Compex RL100-TX", 0),
+PCI_ROM(0x1259, 0xa120, "tulip-a120",  "Tulip 0x1259 0xa120", 0),
+PCI_ROM(0x125b, 0x1400, "ax88140",     "ASIX AX88140", 0),
+PCI_ROM(0x1282, 0x9009, "dm9009",      "Davicom 9009", 0),
 PCI_ROM(0x1282, 0x9100, "dm9100",      "Davicom 9100", 0),
 PCI_ROM(0x1282, 0x9102, "dm9102",      "Davicom 9102", 0),
-PCI_ROM(0x1282, 0x9009, "dm9009",      "Davicom 9009", 0),
 PCI_ROM(0x1282, 0x9132, "dm9132",      "Davicom 9132", 0),
-PCI_ROM(0x1317, 0x0985, "centaur-p",   "ADMtek Centaur-P", 0),
 PCI_ROM(0x1317, 0x0981, "an981",       "ADMtek AN981 Comet", 0),		/* ADMTek Centaur-P (stmicro) */
-PCI_ROM(0x1113, 0x1216, "an983",       "ADMTek AN983 Comet", 0),
-PCI_ROM(0x1317, 0x9511, "an983b",      "ADMTek Comet 983b", 0),
+PCI_ROM(0x1317, 0x0985, "centaur-p",   "ADMtek Centaur-P", 0),
 PCI_ROM(0x1317, 0x1985, "centaur-c",   "ADMTek Centaur-C", 0),
-PCI_ROM(0x8086, 0x0039, "intel21145",  "Intel Tulip", 0),
-PCI_ROM(0x125b, 0x1400, "ax88140",     "ASIX AX88140", 0),
-PCI_ROM(0x11f6, 0x9881, "rl100tx",     "Compex RL100-TX", 0),
-PCI_ROM(0x115d, 0x0003, "xircomtulip", "Xircom Tulip", 0),
-PCI_ROM(0x104a, 0x0981, "tulip-0981",  "Tulip 0x104a 0x0981", 0),
-PCI_ROM(0x104a, 0x2774, "SGThomson-STE10100A", "Tulip 0x104a 0x2774", 0),	/*Modified by Ramesh Chander*/
-PCI_ROM(0x1113, 0x9511, "tulip-9511",  "Tulip 0x1113 0x9511", 0),
-PCI_ROM(0x1186, 0x1561, "tulip-1561",  "Tulip 0x1186 0x1561", 0),
-PCI_ROM(0x1259, 0xa120, "tulip-a120",  "Tulip 0x1259 0xa120", 0),
+PCI_ROM(0x1317, 0x9511, "an983b",      "ADMTek Comet 983b", 0),
 PCI_ROM(0x13d1, 0xab02, "tulip-ab02",  "Tulip 0x13d1 0xab02", 0),
 PCI_ROM(0x13d1, 0xab03, "tulip-ab03",  "Tulip 0x13d1 0xab03", 0),
 PCI_ROM(0x13d1, 0xab08, "tulip-ab08",  "Tulip 0x13d1 0xab08", 0),
@@ -1953,6 +1952,7 @@ PCI_ROM(0x14f1, 0x1803, "lanfinity",   "Conexant LANfinity", 0),
 PCI_ROM(0x1626, 0x8410, "tulip-8410",  "Tulip 0x1626 0x8410", 0),
 PCI_ROM(0x1737, 0xab08, "tulip-1737-ab08","Tulip 0x1737 0xab08", 0),
 PCI_ROM(0x1737, 0xab09, "tulip-ab09",  "Tulip 0x1737 0xab09", 0),
+PCI_ROM(0x8086, 0x0039, "intel21145",  "Intel Tulip", 0),
 };
 
 PCI_DRIVER ( tulip_driver, tulip_nics, PCI_NO_CLASS );
diff --git a/src/hci/commands/sanboot_cmd.c b/src/hci/commands/sanboot_cmd.c
index 3907276a..6ab9e884 100644
--- a/src/hci/commands/sanboot_cmd.c
+++ b/src/hci/commands/sanboot_cmd.c
@@ -47,14 +47,20 @@ struct sanboot_options {
 	int no_describe;
 	/** Keep SAN device */
 	int keep;
-	/** Filename */
+	/** Boot filename */
 	char *filename;
+	/** Required extra filename */
+	char *extra;
+	/** Volume label */
+	char *label;
+	/** UUID */
+	struct uuid_option uuid;
 };
 
 /** "sanboot" option list */
 static union {
-	/* "sanboot" takes all four options */
-	struct option_descriptor sanboot[4];
+	/* "sanboot" takes all options */
+	struct option_descriptor sanboot[7];
 	/* "sanhook" takes only --drive and --no-describe */
 	struct option_descriptor sanhook[2];
 	/* "sanunhook" takes only --drive */
@@ -69,10 +75,15 @@ static union {
 			      struct sanboot_options, keep, parse_flag ),
 		OPTION_DESC ( "filename", 'f', required_argument,
 			      struct sanboot_options, filename, parse_string ),
+		OPTION_DESC ( "extra", 'e', required_argument,
+			      struct sanboot_options, extra, parse_string ),
+		OPTION_DESC ( "label", 'l', required_argument,
+			      struct sanboot_options, label, parse_string ),
+		OPTION_DESC ( "uuid", 'u', required_argument,
+			      struct sanboot_options, uuid, parse_uuid ),
 	},
 };
 
-
 /** "sanhook" command descriptor */
 static struct command_descriptor sanhook_cmd =
 	COMMAND_DESC ( struct sanboot_options, opts.sanhook, 1, MAX_ARGUMENTS,
@@ -100,6 +111,7 @@ static int sanboot_core_exec ( int argc, char **argv,
 			       struct command_descriptor *cmd,
 			       int default_flags, int no_root_path_flags ) {
 	struct sanboot_options opts;
+	struct san_boot_config config;
 	struct uri *uris[argc];
 	int count;
 	int flags;
@@ -124,6 +136,12 @@ static int sanboot_core_exec ( int argc, char **argv,
 		}
 	}
 
+	/* Construct configuration parameters */
+	config.filename = opts.filename;
+	config.extra = opts.extra;
+	config.label = opts.label;
+	config.uuid = opts.uuid.value;
+
 	/* Construct flags */
 	flags = default_flags;
 	if ( opts.no_describe )
@@ -134,7 +152,7 @@ static int sanboot_core_exec ( int argc, char **argv,
 		flags |= no_root_path_flags;
 
 	/* Boot from root path */
-	if ( ( rc = uriboot ( NULL, uris, count, opts.drive, opts.filename,
+	if ( ( rc = uriboot ( NULL, uris, count, opts.drive, &config,
 			      flags ) ) != 0 )
 		goto err_uriboot;
 
diff --git a/src/image/script.c b/src/image/script.c
index 49b35640..9e8566bc 100644
--- a/src/image/script.c
+++ b/src/image/script.c
@@ -219,8 +219,7 @@ static int script_exec ( struct image *image ) {
 static int script_probe ( struct image *image ) {
 	static const char ipxe_magic[] = "#!ipxe";
 	static const char gpxe_magic[] = "#!gpxe";
-	linker_assert ( sizeof ( ipxe_magic ) == sizeof ( gpxe_magic ),
-			magic_size_mismatch );
+	static_assert ( sizeof ( ipxe_magic ) == sizeof ( gpxe_magic ) );
 	char test[ sizeof ( ipxe_magic ) - 1 /* NUL */
 		   + 1 /* terminating space */];
 
diff --git a/src/include/assert.h b/src/include/assert.h
index dd71fa71..01a28785 100644
--- a/src/include/assert.h
+++ b/src/include/assert.h
@@ -56,19 +56,31 @@ assert_printf ( const char *fmt, ... ) asm ( "printf" );
 	} while ( 0 )
 
 /**
- * Assert a condition at link-time.
+ * Assert a condition at build time
  *
- * If the condition is not true, the link will fail with an unresolved
- * symbol (error_symbol).
+ * If the compiler cannot prove that the condition is true, the build
+ * will fail with an error message.
+ */
+#undef static_assert
+#define static_assert(x) _Static_assert( x, #x )
+
+/**
+ * Assert a condition at build time (after dead code elimination)
+ *
+ * If the compiler cannot prove that the condition is true, the build
+ * will fail with an error message.
  *
  * This macro is iPXE-specific.  Do not use this macro in code
  * intended to be portable.
- *
  */
-#define linker_assert( condition, error_symbol )	\
-        if ( ! (condition) ) {				\
-                extern void error_symbol ( void );	\
-                error_symbol();				\
-        }
+#define build_assert( condition )					     \
+	do { 								     \
+		if ( ! (condition) ) {					     \
+			extern void __attribute__ (( warning (		     \
+				"build_assert(" #condition ") failed"	     \
+			) )) _C2 ( build_assert_, __LINE__ ) ( void );	     \
+			_C2 ( build_assert_, __LINE__ ) ();		     \
+		}							     \
+	} while ( 0 )
 
 #endif /* _ASSERT_H */
diff --git a/src/include/ipxe/asn1.h b/src/include/ipxe/asn1.h
index 77429f3a..ac7ea560 100644
--- a/src/include/ipxe/asn1.h
+++ b/src/include/ipxe/asn1.h
@@ -187,6 +187,11 @@ struct asn1_builder_header {
 	ASN1_OID_SINGLE ( 3 ), ASN1_OID_SINGLE ( 2 ),		\
 	ASN1_OID_SINGLE ( 26 )
 
+/** ASN.1 OID for id-x25519 (1.3.101.110) */
+#define ASN1_OID_X25519						\
+	ASN1_OID_INITIAL ( 1, 3 ), ASN1_OID_SINGLE ( 101 ),	\
+	ASN1_OID_SINGLE ( 110 )
+
 /** ASN.1 OID for id-sha256 (2.16.840.1.101.3.4.2.1) */
 #define ASN1_OID_SHA256						\
 	ASN1_OID_INITIAL ( 2, 16 ), ASN1_OID_DOUBLE ( 840 ),	\
@@ -312,6 +317,8 @@ struct asn1_algorithm {
 	struct pubkey_algorithm *pubkey;
 	/** Digest algorithm (if applicable) */
 	struct digest_algorithm *digest;
+	/** Elliptic curve (if applicable) */
+	struct elliptic_curve *curve;
 };
 
 /** ASN.1 OID-identified algorithms */
@@ -390,10 +397,9 @@ asn1_built ( struct asn1_builder *builder ) {
 	} *u = container_of ( builder, typeof ( *u ), builder );
 
 	/* Sanity check */
-	linker_assert ( ( ( const void * ) &u->builder.data ) ==
-			&u->cursor.data, asn1_builder_cursor_data_mismatch );
-	linker_assert ( &u->builder.len == &u->cursor.len,
-			asn1_builder_cursor_len_mismatch );
+	build_assert ( ( ( const void * ) &u->builder.data ) ==
+		       &u->cursor.data );
+	build_assert ( &u->builder.len == &u->cursor.len );
 
 	return &u->cursor;
 }
diff --git a/src/include/ipxe/base16.h b/src/include/ipxe/base16.h
index 8c44da17..c9e430e7 100644
--- a/src/include/ipxe/base16.h
+++ b/src/include/ipxe/base16.h
@@ -12,6 +12,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <stdint.h>
 #include <string.h>
 
+/** Treat separator as optional while decoding */
+#define HEX_DECODE_OPTIONAL 0x80
+
 /**
  * Calculate length of base16-encoded data
  *
diff --git a/src/include/ipxe/bigint.h b/src/include/ipxe/bigint.h
index 2f99f844..3dc344df 100644
--- a/src/include/ipxe/bigint.h
+++ b/src/include/ipxe/bigint.h
@@ -8,6 +8,8 @@
 
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
+#include <assert.h>
+
 /**
  * Define a big-integer type
  *
@@ -177,6 +179,30 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 	} while ( 0 )
 
 /**
+ * Copy big integer
+ *
+ * @v source		Source big integer
+ * @v dest		Destination big integer
+ */
+#define bigint_copy( source, dest ) do {				\
+	build_assert ( sizeof ( *(source) ) == sizeof ( *(dest) ) );	\
+	bigint_shrink ( (source), (dest) );				\
+	} while ( 0 )
+
+/**
+ * Conditionally swap big integers (in constant time)
+ *
+ * @v first		Big integer to be conditionally swapped
+ * @v second		Big integer to be conditionally swapped
+ * @v swap		Swap first and second big integers
+ */
+#define bigint_swap( first, second, swap ) do {				\
+	unsigned int size = bigint_size (first);			\
+	bigint_swap_raw ( (first)->element, (second)->element, size,	\
+			  (swap) );					\
+	} while ( 0 )
+
+/**
  * Multiply big integers
  *
  * @v multiplicand	Big integer to be multiplied
@@ -184,10 +210,11 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * @v result		Big integer to hold result
  */
 #define bigint_multiply( multiplicand, multiplier, result ) do {	\
-	unsigned int size = bigint_size (multiplicand);			\
+	unsigned int multiplicand_size = bigint_size (multiplicand);	\
+	unsigned int multiplier_size = bigint_size (multiplier);	\
 	bigint_multiply_raw ( (multiplicand)->element,			\
-			      (multiplier)->element, (result)->element,	\
-			      size );					\
+			      multiplicand_size, (multiplier)->element,	\
+			      multiplier_size, (result)->element );	\
 	} while ( 0 )
 
 /**
@@ -282,10 +309,13 @@ void bigint_grow_raw ( const bigint_element_t *source0,
 void bigint_shrink_raw ( const bigint_element_t *source0,
 			 unsigned int source_size, bigint_element_t *dest0,
 			 unsigned int dest_size );
+void bigint_swap_raw ( bigint_element_t *first0, bigint_element_t *second0,
+		       unsigned int size, int swap );
 void bigint_multiply_raw ( const bigint_element_t *multiplicand0,
+			   unsigned int multiplicand_size,
 			   const bigint_element_t *multiplier0,
-			   bigint_element_t *result0,
-			   unsigned int size );
+			   unsigned int multiplier_size,
+			   bigint_element_t *result0 );
 void bigint_mod_multiply_raw ( const bigint_element_t *multiplicand0,
 			       const bigint_element_t *multiplier0,
 			       const bigint_element_t *modulus0,
diff --git a/src/include/ipxe/cbc.h b/src/include/ipxe/cbc.h
index 382fc903..f02e5193 100644
--- a/src/include/ipxe/cbc.h
+++ b/src/include/ipxe/cbc.h
@@ -77,19 +77,19 @@ static void _cbc_name ## _setiv ( void *ctx, const void *iv,		\
 				   size_t ivlen ) {			\
 	struct _cbc_name ## _context * _cbc_name ## _ctx = ctx;		\
 	cbc_setiv ( &_cbc_name ## _ctx->raw_ctx, iv, ivlen,		\
-		    &_raw_cipher, &aes_cbc_ctx->cbc_ctx );		\
+		    &_raw_cipher, &_cbc_name ## _ctx->cbc_ctx );	\
 }									\
 static void _cbc_name ## _encrypt ( void *ctx, const void *src,		\
 				    void *dst, size_t len ) {		\
 	struct _cbc_name ## _context * _cbc_name ## _ctx = ctx;		\
 	cbc_encrypt ( &_cbc_name ## _ctx->raw_ctx, src, dst, len,	\
-		      &_raw_cipher, &aes_cbc_ctx->cbc_ctx );		\
+		      &_raw_cipher, &_cbc_name ## _ctx->cbc_ctx );	\
 }									\
 static void _cbc_name ## _decrypt ( void *ctx, const void *src,		\
 				    void *dst, size_t len ) {		\
 	struct _cbc_name ## _context * _cbc_name ## _ctx = ctx;		\
 	cbc_decrypt ( &_cbc_name ## _ctx->raw_ctx, src, dst, len,	\
-		      &_raw_cipher, &aes_cbc_ctx->cbc_ctx );		\
+		      &_raw_cipher, &_cbc_name ## _ctx->cbc_ctx );	\
 }									\
 struct cipher_algorithm _cbc_cipher = {					\
 	.name		= #_cbc_name,					\
diff --git a/src/include/ipxe/crypto.h b/src/include/ipxe/crypto.h
index a15d5eba..a6f43765 100644
--- a/src/include/ipxe/crypto.h
+++ b/src/include/ipxe/crypto.h
@@ -195,34 +195,54 @@ struct pubkey_algorithm {
 			  const void *public_key, size_t public_key_len );
 };
 
-static inline void digest_init ( struct digest_algorithm *digest,
-				 void *ctx ) {
+/** An elliptic curve */
+struct elliptic_curve {
+	/** Curve name */
+	const char *name;
+	/** Key size */
+	size_t keysize;
+	/** Multiply scalar by curve point
+	 *
+	 * @v base		Base point (or NULL to use generator)
+	 * @v scalar		Scalar multiple
+	 * @v result		Result point to fill in
+	 * @ret rc		Return status code
+	 */
+	int ( * multiply ) ( const void *base, const void *scalar,
+			     void *result );
+};
+
+static inline __attribute__ (( always_inline )) void
+digest_init ( struct digest_algorithm *digest, void *ctx ) {
 	digest->init ( ctx );
 }
 
-static inline void digest_update ( struct digest_algorithm *digest,
-				   void *ctx, const void *data, size_t len ) {
+static inline __attribute__ (( always_inline )) void
+digest_update ( struct digest_algorithm *digest, void *ctx,
+		const void *data, size_t len ) {
 	digest->update ( ctx, data, len );
 }
 
-static inline void digest_final ( struct digest_algorithm *digest,
-				  void *ctx, void *out ) {
+static inline __attribute__ (( always_inline )) void
+digest_final ( struct digest_algorithm *digest, void *ctx, void *out ) {
 	digest->final ( ctx, out );
 }
 
-static inline int cipher_setkey ( struct cipher_algorithm *cipher,
-				  void *ctx, const void *key, size_t keylen ) {
+static inline __attribute__ (( always_inline )) int
+cipher_setkey ( struct cipher_algorithm *cipher, void *ctx,
+		const void *key, size_t keylen ) {
 	return cipher->setkey ( ctx, key, keylen );
 }
 
-static inline void cipher_setiv ( struct cipher_algorithm *cipher,
-				  void *ctx, const void *iv, size_t ivlen ) {
+static inline __attribute__ (( always_inline )) void
+cipher_setiv ( struct cipher_algorithm *cipher, void *ctx,
+	       const void *iv, size_t ivlen ) {
 	cipher->setiv ( ctx, iv, ivlen );
 }
 
-static inline void cipher_encrypt ( struct cipher_algorithm *cipher,
-				    void *ctx, const void *src, void *dst,
-				    size_t len ) {
+static inline __attribute__ (( always_inline )) void
+cipher_encrypt ( struct cipher_algorithm *cipher, void *ctx,
+		 const void *src, void *dst, size_t len ) {
 	cipher->encrypt ( ctx, src, dst, len );
 }
 #define cipher_encrypt( cipher, ctx, src, dst, len ) do {		\
@@ -230,9 +250,9 @@ static inline void cipher_encrypt ( struct cipher_algorithm *cipher,
 	cipher_encrypt ( (cipher), (ctx), (src), (dst), (len) );	\
 	} while ( 0 )
 
-static inline void cipher_decrypt ( struct cipher_algorithm *cipher,
-				    void *ctx, const void *src, void *dst,
-				    size_t len ) {
+static inline __attribute__ (( always_inline )) void
+cipher_decrypt ( struct cipher_algorithm *cipher, void *ctx,
+		 const void *src, void *dst, size_t len ) {
 	cipher->decrypt ( ctx, src, dst, len );
 }
 #define cipher_decrypt( cipher, ctx, src, dst, len ) do {		\
@@ -240,68 +260,82 @@ static inline void cipher_decrypt ( struct cipher_algorithm *cipher,
 	cipher_decrypt ( (cipher), (ctx), (src), (dst), (len) );	\
 	} while ( 0 )
 
-static inline void cipher_auth ( struct cipher_algorithm *cipher, void *ctx,
-				 void *auth ) {
+static inline __attribute__ (( always_inline )) void
+cipher_auth ( struct cipher_algorithm *cipher, void *ctx, void *auth ) {
 	cipher->auth ( ctx, auth );
 }
 
-static inline int is_stream_cipher ( struct cipher_algorithm *cipher ) {
+static inline __attribute__ (( always_inline )) int
+is_stream_cipher ( struct cipher_algorithm *cipher ) {
 	return ( cipher->blocksize == 1 );
 }
 
-static inline int is_block_cipher ( struct cipher_algorithm *cipher ) {
+static inline __attribute__ (( always_inline )) int
+is_block_cipher ( struct cipher_algorithm *cipher ) {
 	return ( cipher->blocksize > 1 );
 }
 
-static inline int is_auth_cipher ( struct cipher_algorithm *cipher ) {
+static inline __attribute__ (( always_inline )) int
+is_auth_cipher ( struct cipher_algorithm *cipher ) {
 	return cipher->authsize;
 }
 
-static inline int pubkey_init ( struct pubkey_algorithm *pubkey, void *ctx,
-				const void *key, size_t key_len ) {
+static inline __attribute__ (( always_inline )) int
+pubkey_init ( struct pubkey_algorithm *pubkey, void *ctx,
+	      const void *key, size_t key_len ) {
 	return pubkey->init ( ctx, key, key_len );
 }
 
-static inline size_t pubkey_max_len ( struct pubkey_algorithm *pubkey,
-				      void *ctx ) {
+static inline __attribute__ (( always_inline )) size_t
+pubkey_max_len ( struct pubkey_algorithm *pubkey, void *ctx ) {
 	return pubkey->max_len ( ctx );
 }
 
-static inline int pubkey_encrypt ( struct pubkey_algorithm *pubkey, void *ctx,
-				   const void *data, size_t len, void *out ) {
+static inline __attribute__ (( always_inline )) int
+pubkey_encrypt ( struct pubkey_algorithm *pubkey, void *ctx,
+		 const void *data, size_t len, void *out ) {
 	return pubkey->encrypt ( ctx, data, len, out );
 }
 
-static inline int pubkey_decrypt ( struct pubkey_algorithm *pubkey, void *ctx,
-				   const void *data, size_t len, void *out ) {
+static inline __attribute__ (( always_inline )) int
+pubkey_decrypt ( struct pubkey_algorithm *pubkey, void *ctx,
+		 const void *data, size_t len, void *out ) {
 	return pubkey->decrypt ( ctx, data, len, out );
 }
 
-static inline int pubkey_sign ( struct pubkey_algorithm *pubkey, void *ctx,
-				struct digest_algorithm *digest,
-				const void *value, void *signature ) {
+static inline __attribute__ (( always_inline )) int
+pubkey_sign ( struct pubkey_algorithm *pubkey, void *ctx,
+	      struct digest_algorithm *digest, const void *value,
+	      void *signature ) {
 	return pubkey->sign ( ctx, digest, value, signature );
 }
 
-static inline int pubkey_verify ( struct pubkey_algorithm *pubkey, void *ctx,
-				  struct digest_algorithm *digest,
-				  const void *value, const void *signature,
-				  size_t signature_len ) {
+static inline __attribute__ (( always_inline )) int
+pubkey_verify ( struct pubkey_algorithm *pubkey, void *ctx,
+		struct digest_algorithm *digest, const void *value,
+		const void *signature, size_t signature_len ) {
 	return pubkey->verify ( ctx, digest, value, signature, signature_len );
 }
 
-static inline void pubkey_final ( struct pubkey_algorithm *pubkey, void *ctx ) {
+static inline __attribute__ (( always_inline )) void
+pubkey_final ( struct pubkey_algorithm *pubkey, void *ctx ) {
 	pubkey->final ( ctx );
 }
 
-static inline int pubkey_match ( struct pubkey_algorithm *pubkey,
-				 const void *private_key,
-				 size_t private_key_len, const void *public_key,
-				 size_t public_key_len ) {
+static inline __attribute__ (( always_inline )) int
+pubkey_match ( struct pubkey_algorithm *pubkey,
+	       const void *private_key, size_t private_key_len,
+	       const void *public_key, size_t public_key_len ) {
 	return pubkey->match ( private_key, private_key_len, public_key,
 			       public_key_len );
 }
 
+static inline __attribute__ (( always_inline )) int
+elliptic_multiply ( struct elliptic_curve *curve,
+		    const void *base, const void *scalar, void *result ) {
+	return curve->multiply ( base, scalar, result );
+}
+
 extern void digest_null_init ( void *ctx );
 extern void digest_null_update ( void *ctx, const void *src, size_t len );
 extern void digest_null_final ( void *ctx, void *out );
diff --git a/src/include/ipxe/des.h b/src/include/ipxe/des.h
new file mode 100644
index 00000000..755a90ea
--- /dev/null
+++ b/src/include/ipxe/des.h
@@ -0,0 +1,91 @@
+#ifndef _IPXE_DES_H
+#define _IPXE_DES_H
+
+/** @file
+ *
+ * DES algorithm
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/crypto.h>
+
+/** A DES 32-bit dword value
+ *
+ * DES views data as 64-bit big-endian values, typically handled as a
+ * most-significant "left" half and a least-significant "right" half.
+ */
+union des_dword {
+	/** Raw bytes */
+	uint8_t byte[4];
+	/** 32-bit big-endian dword */
+	uint32_t dword;
+};
+
+/** A DES 64-bit block */
+union des_block {
+	/** Raw bytes */
+	uint8_t byte[8];
+	/** 32-bit big-endian dwords */
+	uint32_t dword[2];
+	/** Named left and right halves */
+	struct {
+		/** Left (most significant) half */
+		union des_dword left;
+		/** Right (least significant) half */
+		union des_dword right;
+	};
+	/** Named "C" and "D" halves */
+	struct {
+		/** "C" (most significant) half */
+		union des_dword c;
+		/** "D" (least significant) half */
+		union des_dword d;
+	};
+};
+
+/** DES blocksize */
+#define DES_BLOCKSIZE sizeof ( union des_block )
+
+/** A DES round key
+ *
+ * A DES round key is a 48-bit value, consumed as 8 groups of 6 bits.
+ * We store these as 8 separate bytes, for simplicity of consumption.
+ */
+union des_round_key {
+	/** Raw bytes */
+	uint8_t byte[8];
+	/** 32-bit big-endian dwords */
+	uint32_t dword[2];
+	/** 6-bit step key byte
+	 *
+	 * There are 8 steps within a DES round (one step per S-box).
+	 * Each step requires six bits of the round key.
+	 *
+	 * As an optimisation, we store the least significant of the 6
+	 * bits in the sign bit of a signed 8-bit value, and the
+	 * remaining 5 bits in the least significant 5 bits of the
+	 * 8-bit value.  See the comments in des_sbox() for further
+	 * details.
+	 */
+	int8_t step[8];
+};
+
+/** Number of DES rounds */
+#define DES_ROUNDS 16
+
+/** DES context */
+struct des_context {
+	/** Round keys */
+	union des_round_key rkey[DES_ROUNDS];
+};
+
+/** DES context size */
+#define DES_CTX_SIZE sizeof ( struct des_context )
+
+extern struct cipher_algorithm des_algorithm;
+extern struct cipher_algorithm des_ecb_algorithm;
+extern struct cipher_algorithm des_cbc_algorithm;
+
+#endif /* _IPXE_DES_H */
diff --git a/src/include/ipxe/eap.h b/src/include/ipxe/eap.h
index e5f60655..a44f01e0 100644
--- a/src/include/ipxe/eap.h
+++ b/src/include/ipxe/eap.h
@@ -12,6 +12,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <stdint.h>
 #include <ipxe/netdevice.h>
 #include <ipxe/timer.h>
+#include <ipxe/tables.h>
 
 /** EAP header */
 struct eap_header {
@@ -26,17 +27,68 @@ struct eap_header {
 /** EAP request */
 #define EAP_CODE_REQUEST 1
 
-/** EAP request */
-struct eap_request {
+/** EAP response */
+#define EAP_CODE_RESPONSE 2
+
+/** EAP request/response message */
+struct eap_message {
 	/** Header */
 	struct eap_header hdr;
 	/** Type */
 	uint8_t type;
+	/** Type data */
+	uint8_t data[0];
 } __attribute__ (( packed ));
 
+/** EAP "no available types" marker */
+#define EAP_TYPE_NONE 0
+
 /** EAP identity */
 #define EAP_TYPE_IDENTITY 1
 
+/** EAP NAK */
+#define EAP_TYPE_NAK 3
+
+/** EAP MD5 challenge request/response */
+#define EAP_TYPE_MD5 4
+
+/** EAP MD5 challenge request/response type data */
+struct eap_md5 {
+	/** Value length */
+	uint8_t len;
+	/** Value */
+	uint8_t value[0];
+} __attribute__ (( packed ));
+
+/** EAP MS-CHAPv2 request/response */
+#define EAP_TYPE_MSCHAPV2 26
+
+/** EAP MS-CHAPv2 request/response type data */
+struct eap_mschapv2 {
+	/** Code
+	 *
+	 * This is in the same namespace as the EAP header's code
+	 * field, but is used to extend the handshake by allowing for
+	 * "success request" and "success response" packets.
+	 */
+	uint8_t code;
+	/** Identifier
+	 *
+	 * This field serves no purposes: it always has the same value
+	 * as the EAP header's identifier field (located 5 bytes
+	 * earlier in the same packet).
+	 */
+	uint8_t id;
+	/** Length
+	 *
+	 * This field serves no purpose: it always has the same value
+	 * as the EAP header's length field (located 5 bytes earlier
+	 * in the same packet), minus the 5 byte length of the EAP
+	 * header.
+	 */
+	uint16_t len;
+} __attribute__ (( packed ));
+
 /** EAP success */
 #define EAP_CODE_SUCCESS 3
 
@@ -47,11 +99,11 @@ struct eap_request {
 union eap_packet {
 	/** Header */
 	struct eap_header hdr;
-	/** Request */
-	struct eap_request req;
+	/** Request/response message */
+	struct eap_message msg;
 };
 
-/** Link block timeout
+/** EAP link block timeout
  *
  * We mark the link as blocked upon receiving a Request-Identity, on
  * the basis that this most likely indicates that the switch will not
@@ -64,12 +116,34 @@ union eap_packet {
  */
 #define EAP_BLOCK_TIMEOUT ( 45 * TICKS_PER_SEC )
 
+/** EAP protocol wait timeout
+ *
+ * In the EAP model, the supplicant is a pure responder.  The model
+ * also defines no acknowledgement response for the final Success or
+ * Failure "requests".  This leaves open the possibility that the
+ * final Success or Failure packet is lost, with the supplicant having
+ * no way to determine the final authentication status.
+ *
+ * Sideband mechanisms such as EAPoL-Start may be used to restart the
+ * entire EAP process, as a (crude) workaround for this protocol flaw.
+ * When expecting to receive a further EAP request (e.g. an
+ * authentication challenge), we may wait for some length of time
+ * before triggering this restart.  Choose a duration that is shorter
+ * than the link block timeout, so that there is no period during
+ * which we erroneously leave the link marked as not blocked.
+ */
+#define EAP_WAIT_TIMEOUT ( EAP_BLOCK_TIMEOUT * 7 / 8 )
+
 /** An EAP supplicant */
 struct eap_supplicant {
 	/** Network device */
 	struct net_device *netdev;
-	/** Authentication outcome is final */
-	int done;
+	/** Flags */
+	uint16_t flags;
+	/** ID for current request/response */
+	uint8_t id;
+	/** Type for current request/response */
+	uint8_t type;
 	/**
 	 * Transmit EAP response
 	 *
@@ -82,6 +156,47 @@ struct eap_supplicant {
 		       const void *data, size_t len );
 };
 
+/** EAP authentication is in progress
+ *
+ * This indicates that we have received an EAP Request-Identity, but
+ * have not yet received a final EAP Success or EAP Failure.
+ */
+#define EAP_FL_ONGOING 0x0001
+
+/** EAP supplicant is passive
+ *
+ * This indicates that the supplicant should not transmit any futher
+ * unsolicited packets (e.g. EAPoL-Start for a supplicant running over
+ * EAPoL).  This could be because authentication has already
+ * completed, or because we are relying upon MAC Authentication Bypass
+ * (MAB) which may have a very long timeout.
+ */
+#define EAP_FL_PASSIVE 0x0002
+
+/** An EAP method */
+struct eap_method {
+	/** Type */
+	uint8_t type;
+	/**
+	 * Handle EAP request
+	 *
+	 * @v supplicant	EAP supplicant
+	 * @v req		Request type data
+	 * @v req_len		Length of request type data
+	 * @ret rc		Return status code
+	 */
+	int ( * rx ) ( struct eap_supplicant *supplicant,
+		       const void *req, size_t req_len );
+};
+
+/** EAP method table */
+#define EAP_METHODS __table ( struct eap_method, "eap_methods" )
+
+/** Declare an EAP method */
+#define __eap_method __table_entry ( EAP_METHODS, 01 )
+
+extern int eap_tx_response ( struct eap_supplicant *supplicant,
+			     const void *rsp, size_t rsp_len );
 extern int eap_rx ( struct eap_supplicant *supplicant,
 		    const void *data, size_t len );
 
diff --git a/src/include/ipxe/eapol.h b/src/include/ipxe/eapol.h
index d4ea3920..dcf39294 100644
--- a/src/include/ipxe/eapol.h
+++ b/src/include/ipxe/eapol.h
@@ -42,11 +42,16 @@ struct eapol_supplicant {
 	struct eap_supplicant eap;
 	/** EAPoL-Start retransmission timer */
 	struct retry_timer timer;
+	/** EAPoL-Start transmission count */
+	unsigned int count;
 };
 
 /** Delay between EAPoL-Start packets */
 #define EAPOL_START_INTERVAL ( 2 * TICKS_PER_SEC )
 
+/** Maximum number of EAPoL-Start packets to transmit */
+#define EAPOL_START_COUNT 3
+
 /** An EAPoL handler */
 struct eapol_handler {
 	/** Type */
diff --git a/src/include/ipxe/ecam.h b/src/include/ipxe/ecam.h
index 683d613a..ff08aee5 100644
--- a/src/include/ipxe/ecam.h
+++ b/src/include/ipxe/ecam.h
@@ -50,6 +50,8 @@ struct ecam_mapping {
 	struct pci_range range;
 	/** MMIO base address */
 	void *regs;
+	/** Mapping result */
+	int rc;
 };
 
 extern struct pci_api ecam_api;
diff --git a/src/include/ipxe/efi/Base.h b/src/include/ipxe/efi/Base.h
index e76013c1..46c31a3b 100644
--- a/src/include/ipxe/efi/Base.h
+++ b/src/include/ipxe/efi/Base.h
@@ -1232,6 +1232,11 @@ typedef UINTN RETURN_STATUS;
 #define RETURN_COMPROMISED_DATA  ENCODE_ERROR (33)
 
 ///
+/// There is an address conflict address allocation.
+///
+#define RETURN_IP_ADDRESS_CONFLICT  ENCODE_ERROR (34)
+
+///
 /// A HTTP error occurred during the network operation.
 ///
 #define RETURN_HTTP_ERROR  ENCODE_ERROR (35)
@@ -1270,6 +1275,11 @@ typedef UINTN RETURN_STATUS;
 ///
 #define RETURN_WARN_FILE_SYSTEM  ENCODE_WARNING (6)
 
+///
+/// The operation will be processed across a system reset.
+///
+#define RETURN_WARN_RESET_REQUIRED  ENCODE_WARNING (7)
+
 /**
   Returns a 16-bit signature built from 2 ASCII characters.
 
diff --git a/src/include/ipxe/efi/Guid/FileInfo.h b/src/include/ipxe/efi/Guid/FileInfo.h
index 4fc9e860..62c5f4c0 100644
--- a/src/include/ipxe/efi/Guid/FileInfo.h
+++ b/src/include/ipxe/efi/Guid/FileInfo.h
@@ -49,6 +49,7 @@ typedef struct {
   UINT64      Attribute;
   ///
   /// The Null-terminated name of the file.
+  /// For a root directory, the name is an empty string.
   ///
   CHAR16      FileName[1];
 } EFI_FILE_INFO;
diff --git a/src/include/ipxe/efi/IndustryStandard/Acpi30.h b/src/include/ipxe/efi/IndustryStandard/Acpi30.h
index c7dfd5c7..ff82bf20 100644
--- a/src/include/ipxe/efi/IndustryStandard/Acpi30.h
+++ b/src/include/ipxe/efi/IndustryStandard/Acpi30.h
@@ -19,6 +19,20 @@ FILE_LICENCE ( BSD2_PATENT );
 
 #define ACPI_EXTENDED_ADDRESS_SPACE_DESCRIPTOR  0x8B
 
+///
+/// C-state Coordination Types
+/// See s8.4.2.2 _CSD (C-State Dependency)
+///
+#define ACPI_AML_COORD_TYPE_SW_ALL  0xFC
+#define ACPI_AML_COORD_TYPE_SW_ANY  0xFD
+#define ACPI_AML_COORD_TYPE_HW_ALL  0xFE
+
+///
+/// _PSD Revision for ACPI 3.0
+// See s8.4.4.5 _PSD (P-State Dependency)
+///
+#define EFI_ACPI_3_0_AML_PSD_REVISION  0
+
 //
 // Ensure proper structure formats
 //
diff --git a/src/include/ipxe/efi/IndustryStandard/Acpi40.h b/src/include/ipxe/efi/IndustryStandard/Acpi40.h
index f6c70d74..97b81703 100644
--- a/src/include/ipxe/efi/IndustryStandard/Acpi40.h
+++ b/src/include/ipxe/efi/IndustryStandard/Acpi40.h
@@ -12,6 +12,11 @@ FILE_LICENCE ( BSD2_PATENT );
 
 #include <ipxe/efi/IndustryStandard/Acpi30.h>
 
+///
+/// _PSD Revision for ACPI 4.0
+///
+#define EFI_ACPI_4_0_AML_PSD_REVISION  0
+
 //
 // Ensure proper structure formats
 //
diff --git a/src/include/ipxe/efi/IndustryStandard/Acpi50.h b/src/include/ipxe/efi/IndustryStandard/Acpi50.h
index 7d57b9ff..2addcb00 100644
--- a/src/include/ipxe/efi/IndustryStandard/Acpi50.h
+++ b/src/include/ipxe/efi/IndustryStandard/Acpi50.h
@@ -25,6 +25,16 @@ FILE_LICENCE ( BSD2_PATENT );
 #define ACPI_GPIO_CONNECTION_DESCRIPTOR                0x8C
 #define ACPI_GENERIC_SERIAL_BUS_CONNECTION_DESCRIPTOR  0x8E
 
+///
+/// _PSD Revision for ACPI 5.0
+///
+#define EFI_ACPI_5_0_AML_PSD_REVISION  0
+
+///
+/// _CPC Revision for ACPI 5.0
+///
+#define EFI_ACPI_5_0_AML_CPC_REVISION  1
+
 #pragma pack(1)
 
 ///
diff --git a/src/include/ipxe/efi/IndustryStandard/Acpi51.h b/src/include/ipxe/efi/IndustryStandard/Acpi51.h
index 49bb972e..a2079ecc 100644
--- a/src/include/ipxe/efi/IndustryStandard/Acpi51.h
+++ b/src/include/ipxe/efi/IndustryStandard/Acpi51.h
@@ -15,6 +15,16 @@ FILE_LICENCE ( BSD2_PATENT );
 
 #include <ipxe/efi/IndustryStandard/Acpi50.h>
 
+///
+/// _PSD Revision for ACPI 5.1
+///
+#define EFI_ACPI_5_1_AML_PSD_REVISION  0
+
+///
+/// _CPC Revision for ACPI 5.1
+///
+#define EFI_ACPI_5_1_AML_CPC_REVISION  2
+
 //
 // Ensure proper structure formats
 //
diff --git a/src/include/ipxe/efi/IndustryStandard/Acpi60.h b/src/include/ipxe/efi/IndustryStandard/Acpi60.h
index 9bd821c7..c8d99214 100644
--- a/src/include/ipxe/efi/IndustryStandard/Acpi60.h
+++ b/src/include/ipxe/efi/IndustryStandard/Acpi60.h
@@ -14,6 +14,16 @@ FILE_LICENCE ( BSD2_PATENT );
 
 #include <ipxe/efi/IndustryStandard/Acpi51.h>
 
+///
+/// _PSD Revision for ACPI 6.0
+///
+#define EFI_ACPI_6_0_AML_PSD_REVISION  0
+
+///
+/// _CPC Revision for ACPI 6.0
+///
+#define EFI_ACPI_6_0_AML_CPC_REVISION  2
+
 //
 // Ensure proper structure formats
 //
diff --git a/src/include/ipxe/efi/IndustryStandard/PeImage.h b/src/include/ipxe/efi/IndustryStandard/PeImage.h
index 401e961c..c1f1a09c 100644
--- a/src/include/ipxe/efi/IndustryStandard/PeImage.h
+++ b/src/include/ipxe/efi/IndustryStandard/PeImage.h
@@ -4,7 +4,7 @@
   EFI_IMAGE_NT_HEADERS64 is for PE32+.
 
   This file is coded to the Visual Studio, Microsoft Portable Executable and
-  Common Object File Format Specification, Revision 8.3 - February 6, 2013.
+  Common Object File Format Specification, Revision 9.3 - December 29, 2015.
   This file also includes some definitions in PI Specification, Revision 1.0.
 
 Copyright (c) 2006 - 2018, Intel Corporation. All rights reserved.<BR>
@@ -271,6 +271,21 @@ typedef struct {
 #define EFI_IMAGE_SUBSYSTEM_OS2_CUI      5
 #define EFI_IMAGE_SUBSYSTEM_POSIX_CUI    7
 
+//
+// DLL Characteristics
+//
+#define IMAGE_DLLCHARACTERISTICS_HIGH_ENTROPY_VA        0x0020
+#define IMAGE_DLLCHARACTERISTICS_DYNAMIC_BASE           0x0040
+#define IMAGE_DLLCHARACTERISTICS_FORCE_INTEGRITY        0x0080
+#define IMAGE_DLLCHARACTERISTICS_NX_COMPAT              0x0100
+#define IMAGE_DLLCHARACTERISTICS_NO_ISOLATION           0x0200
+#define IMAGE_DLLCHARACTERISTICS_NO_SEH                 0x0400
+#define IMAGE_DLLCHARACTERISTICS_NO_BIND                0x0800
+#define IMAGE_DLLCHARACTERISTICS_APPCONTAINER           0x1000
+#define IMAGE_DLLCHARACTERISTICS_WDM_DRIVER             0x2000
+#define IMAGE_DLLCHARACTERISTICS_GUARD_CF               0x4000
+#define IMAGE_DLLCHARACTERISTICS_TERMINAL_SERVER_AWARE  0x8000
+
 ///
 /// Length of ShortName.
 ///
@@ -680,9 +695,6 @@ typedef struct {
   //
 } EFI_IMAGE_DEBUG_CODEVIEW_MTOC_ENTRY;
 
-// avoid conflict with windows header files
-#ifndef RUNTIME_FUNCTION_INDIRECT
-
 //
 // .pdata entries for X64
 //
@@ -692,8 +704,6 @@ typedef struct {
   UINT32    UnwindInfoAddress;
 } RUNTIME_FUNCTION;
 
-#endif
-
 typedef struct {
   UINT8    Version             : 3;
   UINT8    Flags               : 5;
diff --git a/src/include/ipxe/efi/Library/BaseLib.h b/src/include/ipxe/efi/Library/BaseLib.h
index e17f3da2..16ea35cd 100644
--- a/src/include/ipxe/efi/Library/BaseLib.h
+++ b/src/include/ipxe/efi/Library/BaseLib.h
@@ -184,11 +184,21 @@ RiscVSetSupervisorAddressTranslationRegister (
   );
 
 UINT64
+RiscVGetSupervisorAddressTranslationRegister (
+  VOID
+  );
+
+UINT64
 RiscVReadTimer (
   VOID
   );
 
 VOID
+RiscVSetSupervisorTimeCompareRegister (
+  IN UINT64
+  );
+
+VOID
 RiscVEnableTimerInterrupt (
   VOID
   );
@@ -203,6 +213,59 @@ RiscVClearPendingTimerInterrupt (
   VOID
   );
 
+/**
+  RISC-V invalidate instruction cache.
+
+**/
+VOID
+EFIAPI
+RiscVInvalidateInstCacheFenceAsm (
+  VOID
+  );
+
+/**
+  RISC-V invalidate data cache.
+
+**/
+VOID
+EFIAPI
+RiscVInvalidateDataCacheFenceAsm (
+  VOID
+  );
+
+/**
+  RISC-V flush cache block. Atomically perform a clean operation
+  followed by an invalidate operation
+
+**/
+VOID
+EFIAPI
+RiscVCpuCacheFlushCmoAsm (
+  IN UINTN
+  );
+
+/**
+Perform a write transfer to another cache or to memory if the
+data in the copy of the cache block have been modified by a store
+operation
+
+**/
+VOID
+EFIAPI
+RiscVCpuCacheCleanCmoAsm (
+  IN UINTN
+  );
+
+/**
+Deallocate the copy of the cache block
+
+**/
+VOID
+EFIAPI
+RiscVCpuCacheInvalCmoAsm (
+  IN UINTN
+  );
+
 #endif // defined (MDE_CPU_RISCV64)
 
 #if defined (MDE_CPU_LOONGARCH64)
@@ -226,6 +289,227 @@ typedef struct {
 
 #define BASE_LIBRARY_JUMP_BUFFER_ALIGNMENT  8
 
+/*
+ * Set the exception base address for LoongArch.
+ *
+ * @param  ExceptionBaseAddress   The exception base address, must be aligned greater than or qeual to 4K .
+ */
+VOID
+SetExceptionBaseAddress (
+  IN UINT64
+  );
+
+/*
+ * Set the TlbRebase address for LoongArch.
+ *
+ * @param  TlbRebaseAddress   The TlbRebase address, must be aligned greater than or qeual to 4K .
+ */
+VOID
+SetTlbRebaseAddress (
+  IN UINT64
+  );
+
+/**
+  Enables local CPU interrupts.
+
+  @param  Needs to enable local interrupt bit.
+**/
+VOID
+EnableLocalInterrupts (
+  IN UINT16
+  );
+
+/**
+  Disables local CPU interrupts.
+
+  @param  Needs to disable local interrupt bit.
+**/
+VOID
+DisableLocalInterrupts (
+  IN UINT16
+  );
+
+/**
+  Read CPUCFG register.
+
+  @param  Index  Specifies the register number of the CPUCFG to read the data.
+  @param  Data   A pointer to the variable used to store the CPUCFG register value.
+**/
+VOID
+AsmCpucfg (
+  IN  UINT32  Index,
+  OUT UINT32  *Data
+  );
+
+/**
+  Gets the timer count value.
+
+  @param[] VOID
+  @retval  timer count value.
+
+**/
+UINTN
+AsmReadStableCounter (
+  VOID
+  );
+
+/**
+  CSR read operation.
+
+  @param[in]  Select   CSR read instruction select values.
+
+  @return     The return value of csrrd instruction, return -1 means no CSR instruction
+              is found.
+**/
+UINTN
+CsrRead (
+  IN UINT16  Select
+  );
+
+/**
+  CSR write operation.
+
+  @param[in]  Select   CSR write instruction select values.
+  @param[in]  Value    The csrwr will write the value.
+
+  @return     The return value of csrwr instruction, that is, store the old value of
+              the register, return -1 means no CSR instruction is found.
+**/
+UINTN
+CsrWrite (
+  IN UINT16  Select,
+  IN UINTN   Value
+  );
+
+/**
+  CSR exchange operation.
+
+  @param[in]  Select   CSR exchange instruction select values.
+  @param[in]  Value    The csrxchg will write the value.
+  @param[in]  Mask     The csrxchg mask value.
+
+  @return     The return value of csrxchg instruction, that is, store the old value of
+              the register, return -1 means no CSR instruction is found.
+**/
+UINTN
+CsrXChg (
+  IN UINT16  Select,
+  IN UINTN   Value,
+  IN UINTN   Mask
+  );
+
+/**
+  IO CSR read byte operation.
+
+  @param[in]  Select   IO CSR read instruction select values.
+
+  @return     The return value of iocsrrd.b instruction.
+
+**/
+UINT8
+IoCsrRead8 (
+  IN UINTN  Select
+  );
+
+/**
+  IO CSR read half word operation.
+
+  @param[in]  Select   IO CSR read instruction select values.
+
+  @return     The return value of iocsrrd.h instruction.
+
+**/
+UINT16
+IoCsrRead16 (
+  IN UINTN  Select
+  );
+
+/**
+  IO CSR read word operation.
+
+  @param[in]  Select   IO CSR read instruction select values.
+
+  @return     The return value of iocsrrd.w instruction.
+
+**/
+UINT32
+IoCsrRead32 (
+  IN UINTN  Select
+  );
+
+/**
+  IO CSR read double word operation. Only for LoongArch64.
+
+  @param[in]  Select   IO CSR read instruction select values.
+
+  @return     The return value of iocsrrd.d instruction.
+
+**/
+UINT64
+IoCsrRead64 (
+  IN UINTN  Select
+  );
+
+/**
+  IO CSR write byte operation.
+
+  @param[in]  Select   IO CSR write instruction select values.
+  @param[in]  Value    The iocsrwr.b will write the value.
+
+  @return     VOID.
+
+**/
+VOID
+IoCsrWrite8 (
+  IN UINTN  Select,
+  IN UINT8  Value
+  );
+
+/**
+  IO CSR write half word operation.
+
+  @param[in]  Select   IO CSR write instruction select values.
+  @param[in]  Value    The iocsrwr.h will write the value.
+
+  @return     VOID.
+
+**/
+VOID
+IoCsrWrite16 (
+  IN UINTN   Select,
+  IN UINT16  Value
+  );
+
+/**
+  IO CSR write word operation.
+
+  @param[in]  Select   IO CSR write instruction select values.
+  @param[in]  Value    The iocsrwr.w will write the value.
+
+  @return     VOID.
+
+**/
+VOID
+IoCsrWrite32 (
+  IN UINTN   Select,
+  IN UINT32  Value
+  );
+
+/**
+  IO CSR write double word operation. Only for LoongArch64.
+
+  @param[in]  Select   IO CSR write instruction select values.
+  @param[in]  Value    The iocsrwr.d will write the value.
+
+  @return     VOID.
+
+**/
+VOID
+IoCsrWrite64 (
+  IN UINTN   Select,
+  IN UINT64  Value
+  );
+
 #endif // defined (MDE_CPU_LOONGARCH64)
 
 //
@@ -4596,6 +4880,11 @@ CalculateCrc16Ansi (
   IN  UINT16      InitialValue
   );
 
+//
+// Initial value for the CRC16-ANSI algorithm, when no prior checksum has been calculated.
+//
+#define CRC16ANSI_INIT  0xffff
+
 /**
    Calculates the CRC32c checksum of the given buffer.
 
diff --git a/src/include/ipxe/efi/Pi/PiStatusCode.h b/src/include/ipxe/efi/Pi/PiStatusCode.h
index 4375f704..427e5061 100644
--- a/src/include/ipxe/efi/Pi/PiStatusCode.h
+++ b/src/include/ipxe/efi/Pi/PiStatusCode.h
@@ -365,6 +365,7 @@ typedef struct {
 #define EFI_PERIPHERAL_LCD_DEVICE       (EFI_PERIPHERAL | 0x000B0000)
 #define EFI_PERIPHERAL_NETWORK          (EFI_PERIPHERAL | 0x000C0000)
 #define EFI_PERIPHERAL_DOCKING          (EFI_PERIPHERAL | 0x000D0000)
+#define EFI_PERIPHERAL_TPM              (EFI_PERIPHERAL | 0x000E0000)
 ///@}
 
 ///
@@ -967,26 +968,27 @@ typedef struct {
 /// These are shared by all subclasses.
 ///
 ///@{
-#define EFI_SW_EC_NON_SPECIFIC            0x00000000
-#define EFI_SW_EC_LOAD_ERROR              0x00000001
-#define EFI_SW_EC_INVALID_PARAMETER       0x00000002
-#define EFI_SW_EC_UNSUPPORTED             0x00000003
-#define EFI_SW_EC_INVALID_BUFFER          0x00000004
-#define EFI_SW_EC_OUT_OF_RESOURCES        0x00000005
-#define EFI_SW_EC_ABORTED                 0x00000006
-#define EFI_SW_EC_ILLEGAL_SOFTWARE_STATE  0x00000007
-#define EFI_SW_EC_ILLEGAL_HARDWARE_STATE  0x00000008
-#define EFI_SW_EC_START_ERROR             0x00000009
-#define EFI_SW_EC_BAD_DATE_TIME           0x0000000A
-#define EFI_SW_EC_CFG_INVALID             0x0000000B
-#define EFI_SW_EC_CFG_CLR_REQUEST         0x0000000C
-#define EFI_SW_EC_CFG_DEFAULT             0x0000000D
-#define EFI_SW_EC_PWD_INVALID             0x0000000E
-#define EFI_SW_EC_PWD_CLR_REQUEST         0x0000000F
-#define EFI_SW_EC_PWD_CLEARED             0x00000010
-#define EFI_SW_EC_EVENT_LOG_FULL          0x00000011
-#define EFI_SW_EC_WRITE_PROTECTED         0x00000012
-#define EFI_SW_EC_FV_CORRUPTED            0x00000013
+#define EFI_SW_EC_NON_SPECIFIC             0x00000000
+#define EFI_SW_EC_LOAD_ERROR               0x00000001
+#define EFI_SW_EC_INVALID_PARAMETER        0x00000002
+#define EFI_SW_EC_UNSUPPORTED              0x00000003
+#define EFI_SW_EC_INVALID_BUFFER           0x00000004
+#define EFI_SW_EC_OUT_OF_RESOURCES         0x00000005
+#define EFI_SW_EC_ABORTED                  0x00000006
+#define EFI_SW_EC_ILLEGAL_SOFTWARE_STATE   0x00000007
+#define EFI_SW_EC_ILLEGAL_HARDWARE_STATE   0x00000008
+#define EFI_SW_EC_START_ERROR              0x00000009
+#define EFI_SW_EC_BAD_DATE_TIME            0x0000000A
+#define EFI_SW_EC_CFG_INVALID              0x0000000B
+#define EFI_SW_EC_CFG_CLR_REQUEST          0x0000000C
+#define EFI_SW_EC_CFG_DEFAULT              0x0000000D
+#define EFI_SW_EC_PWD_INVALID              0x0000000E
+#define EFI_SW_EC_PWD_CLR_REQUEST          0x0000000F
+#define EFI_SW_EC_PWD_CLEARED              0x00000010
+#define EFI_SW_EC_EVENT_LOG_FULL           0x00000011
+#define EFI_SW_EC_WRITE_PROTECTED          0x00000012
+#define EFI_SW_EC_FV_CORRUPTED             0x00000013
+#define EFI_SW_EC_INCONSISTENT_MEMORY_MAP  0x00000014
 ///@}
 
 //
diff --git a/src/include/ipxe/efi/Protocol/DebugSupport.h b/src/include/ipxe/efi/Protocol/DebugSupport.h
index 453ea975..8f930e33 100644
--- a/src/include/ipxe/efi/Protocol/DebugSupport.h
+++ b/src/include/ipxe/efi/Protocol/DebugSupport.h
@@ -685,6 +685,20 @@ typedef struct {
 //
 // LoongArch processor exception types.
 //
+// The exception types is located in the CSR ESTAT
+// register offset 16 bits, width 6 bits.
+//
+// If you want to register an exception hook, you can
+// shfit the number left by 16 bits, and the exception
+// handler will know the types.
+//
+// For example:
+// mCpu->CpuRegisterInterruptHandler (
+//         mCpu,
+//         (EXCEPT_LOONGARCH_PPI << CSR_ESTAT_EXC_SHIFT),
+//         PpiExceptionHandler
+//         );
+//
 #define EXCEPT_LOONGARCH_INT   0
 #define EXCEPT_LOONGARCH_PIL   1
 #define EXCEPT_LOONGARCH_PIS   2
diff --git a/src/include/ipxe/efi/Protocol/FormBrowser2.h b/src/include/ipxe/efi/Protocol/FormBrowser2.h
index b1c0d200..5e6f940b 100644
--- a/src/include/ipxe/efi/Protocol/FormBrowser2.h
+++ b/src/include/ipxe/efi/Protocol/FormBrowser2.h
@@ -57,6 +57,7 @@ typedef UINTN EFI_BROWSER_ACTION_REQUEST;
 #define EFI_BROWSER_ACTION_REQUEST_FORM_APPLY         6
 #define EFI_BROWSER_ACTION_REQUEST_FORM_DISCARD       7
 #define EFI_BROWSER_ACTION_REQUEST_RECONNECT          8
+#define EFI_BROWSER_ACTION_REQUEST_QUESTION_APPLY     9
 
 /**
   Initialize the browser to display the specified configuration forms.
@@ -140,10 +141,13 @@ EFI_STATUS
   @retval EFI_SUCCESS           The results have been distributed or are
                                 awaiting distribution.
 
-  @retval EFI_OUT_OF_RESOURCES  The ResultsDataSize specified
+  @retval EFI_BUFFER_TOO_SMALL  The ResultsDataSize specified
                                 was too small to contain the
                                 results data.
 
+  @retval EFI_UNSUPPORTED       Uncommitted browser state is not available
+                                at the current stage of execution.
+
 **/
 typedef
 EFI_STATUS
diff --git a/src/include/ipxe/efi/Protocol/HiiConfigAccess.h b/src/include/ipxe/efi/Protocol/HiiConfigAccess.h
index beae0820..aaa51a31 100644
--- a/src/include/ipxe/efi/Protocol/HiiConfigAccess.h
+++ b/src/include/ipxe/efi/Protocol/HiiConfigAccess.h
@@ -104,9 +104,16 @@ typedef UINTN EFI_BROWSER_ACTION;
                                   string.
 
   @retval EFI_INVALID_PARAMETER   Unknown name. Progress points
-                                  to the & before the name in
+                                  to the "&" before the name in
                                   question.
 
+  @retval EFI_INVALID_PARAMETER   If Results or Progress is NULL.
+
+  @retval EFI_ACCESS_DENIED       The action violated a system policy.
+
+  @retval EFI_DEVICE_ERROR        Failed to extract the current configuration
+                                  for one or more named elements.
+
 **/
 typedef
 EFI_STATUS
diff --git a/src/include/ipxe/efi/Protocol/Rng.h b/src/include/ipxe/efi/Protocol/Rng.h
index 87c5c0ed..92d648be 100644
--- a/src/include/ipxe/efi/Protocol/Rng.h
+++ b/src/include/ipxe/efi/Protocol/Rng.h
@@ -69,6 +69,15 @@ typedef EFI_GUID EFI_RNG_ALGORITHM;
   { \
     0xe43176d7, 0xb6e8, 0x4827, {0xb7, 0x84, 0x7f, 0xfd, 0xc4, 0xb6, 0x85, 0x61 } \
   }
+///
+/// The Arm Architecture states the RNDR that the DRBG algorithm should be compliant
+/// with NIST SP800-90A, while not mandating a particular algorithm, so as to be
+/// inclusive of different geographies.
+///
+#define EFI_RNG_ALGORITHM_ARM_RNDR \
+  { \
+    0x43d2fde3, 0x9d4e, 0x4d79,  {0x02, 0x96, 0xa8, 0x9b, 0xca, 0x78, 0x08, 0x41} \
+  }
 
 /**
   Returns information about the random number generation implementation.
@@ -148,5 +157,6 @@ extern EFI_GUID  gEfiRngAlgorithmSp80090Ctr256Guid;
 extern EFI_GUID  gEfiRngAlgorithmX9313DesGuid;
 extern EFI_GUID  gEfiRngAlgorithmX931AesGuid;
 extern EFI_GUID  gEfiRngAlgorithmRaw;
+extern EFI_GUID  gEfiRngAlgorithmArmRndr;
 
 #endif
diff --git a/src/include/ipxe/efi/Protocol/Tcp6.h b/src/include/ipxe/efi/Protocol/Tcp6.h
index eed2f7cc..ddceaaf9 100644
--- a/src/include/ipxe/efi/Protocol/Tcp6.h
+++ b/src/include/ipxe/efi/Protocol/Tcp6.h
@@ -194,12 +194,12 @@ typedef struct {
   BOOLEAN    EnableNagle;
   ///
   /// Set it to TRUE to enable TCP timestamps option as defined in
-  /// RFC1323. Set to FALSE to disable it.
+  /// RFC7323. Set to FALSE to disable it.
   ///
   BOOLEAN    EnableTimeStamp;
   ///
   /// Set it to TRUE to enable TCP window scale option as defined in
-  /// RFC1323. Set it to FALSE to disable it.
+  /// RFC7323. Set it to FALSE to disable it.
   ///
   BOOLEAN    EnableWindowScaling;
   ///
diff --git a/src/include/ipxe/efi/Uefi/UefiBaseType.h b/src/include/ipxe/efi/Uefi/UefiBaseType.h
index 04927599..bf3aa9bb 100644
--- a/src/include/ipxe/efi/Uefi/UefiBaseType.h
+++ b/src/include/ipxe/efi/Uefi/UefiBaseType.h
@@ -143,6 +143,7 @@ typedef union {
 #define EFI_END_OF_FILE           RETURN_END_OF_FILE
 #define EFI_INVALID_LANGUAGE      RETURN_INVALID_LANGUAGE
 #define EFI_COMPROMISED_DATA      RETURN_COMPROMISED_DATA
+#define EFI_IP_ADDRESS_CONFLICT   RETURN_IP_ADDRESS_CONFLICT
 #define EFI_HTTP_ERROR            RETURN_HTTP_ERROR
 
 #define EFI_WARN_UNKNOWN_GLYPH     RETURN_WARN_UNKNOWN_GLYPH
@@ -151,6 +152,7 @@ typedef union {
 #define EFI_WARN_BUFFER_TOO_SMALL  RETURN_WARN_BUFFER_TOO_SMALL
 #define EFI_WARN_STALE_DATA        RETURN_WARN_STALE_DATA
 #define EFI_WARN_FILE_SYSTEM       RETURN_WARN_FILE_SYSTEM
+#define EFI_WARN_RESET_REQUIRED    RETURN_WARN_RESET_REQUIRED
 ///@}
 
 ///
diff --git a/src/include/ipxe/efi/Uefi/UefiSpec.h b/src/include/ipxe/efi/Uefi/UefiSpec.h
index e5a32d88..cc166fc3 100644
--- a/src/include/ipxe/efi/Uefi/UefiSpec.h
+++ b/src/include/ipxe/efi/Uefi/UefiSpec.h
@@ -113,6 +113,21 @@ typedef enum {
 #define EFI_MEMORY_RUNTIME  0x8000000000000000ULL
 
 //
+// If this flag is set, the memory region is
+// described with additional ISA-specific memory attributes
+// as specified in EFI_MEMORY_ISA_MASK.
+//
+#define EFI_MEMORY_ISA_VALID  0x4000000000000000ULL
+
+//
+// Defines the bits reserved for describing optional ISA-specific cacheability
+// attributes that are not covered by the standard UEFI Memory Attributes cacheability
+// bits (EFI_MEMORY_UC, EFI_MEMORY_WC, EFI_MEMORY_WT, EFI_MEMORY_WB and EFI_MEMORY_UCE).
+// See Calling Conventions for further ISA-specific enumeration of these bits.
+//
+#define EFI_MEMORY_ISA_MASK  0x0FFFF00000000000ULL
+
+//
 // Attributes bitmasks, grouped by type
 //
 #define EFI_CACHE_ATTRIBUTE_MASK   (EFI_MEMORY_UC | EFI_MEMORY_WC | EFI_MEMORY_WT | EFI_MEMORY_WB | EFI_MEMORY_UCE | EFI_MEMORY_WP)
@@ -307,6 +322,9 @@ EFI_STATUS
                                 map that requires a mapping.
   @retval EFI_NOT_FOUND         A virtual address was supplied for an address that is not found
                                 in the memory map.
+  @retval EFI_UNSUPPORTED       This call is not supported by this platform at the time the call is made.
+                                The platform should describe this runtime service as unsupported at runtime
+                                via an EFI_RT_PROPERTIES_TABLE configuration table.
 
 **/
 typedef
@@ -397,11 +415,14 @@ EFI_STATUS
                                      for the new virtual address mappings being applied.
 
   @retval EFI_SUCCESS           The pointer pointed to by Address was modified.
-  @retval EFI_INVALID_PARAMETER 1) Address is NULL.
-                                2) *Address is NULL and DebugDisposition does
-                                not have the EFI_OPTIONAL_PTR bit set.
   @retval EFI_NOT_FOUND         The pointer pointed to by Address was not found to be part
                                 of the current memory map. This is normally fatal.
+  @retval EFI_INVALID_PARAMETER Address is NULL.
+  @retval EFI_INVALID_PARAMETER *Address is NULL and DebugDisposition does
+                                not have the EFI_OPTIONAL_PTR bit set.
+  @retval EFI_UNSUPPORTED       This call is not supported by this platform at the time the call is made.
+                                The platform should describe this runtime service as unsupported at runtime
+                                via an EFI_RT_PROPERTIES_TABLE configuration table.
 
 **/
 typedef
@@ -666,6 +687,10 @@ VOID
   @retval EFI_INVALID_PARAMETER  The DataSize is not too small and Data is NULL.
   @retval EFI_DEVICE_ERROR       The variable could not be retrieved due to a hardware error.
   @retval EFI_SECURITY_VIOLATION The variable could not be retrieved due to an authentication failure.
+  @retval EFI_UNSUPPORTED        After ExitBootServices() has been called, this return code may be returned
+                                 if no variable storage is supported. The platform should describe this
+                                 runtime service as unsupported at runtime via an EFI_RT_PROPERTIES_TABLE
+                                 configuration table.
 
 **/
 typedef
@@ -702,6 +727,10 @@ EFI_STATUS
   @retval EFI_INVALID_PARAMETER Null-terminator is not found in the first VariableNameSize bytes of
                                 the input VariableName buffer.
   @retval EFI_DEVICE_ERROR      The variable could not be retrieved due to a hardware error.
+  @retval EFI_UNSUPPORTED       After ExitBootServices() has been called, this return code may be returned
+                                if no variable storage is supported. The platform should describe this
+                                runtime service as unsupported at runtime via an EFI_RT_PROPERTIES_TABLE
+                                configuration table.
 
 **/
 typedef
@@ -744,6 +773,9 @@ EFI_STATUS
                                  but the AuthInfo does NOT pass the validation check carried out by the firmware.
 
   @retval EFI_NOT_FOUND          The variable trying to be updated or deleted was not found.
+  @retval EFI_UNSUPPORTED        This call is not supported by this platform at the time the call is made.
+                                 The platform should describe this runtime service as unsupported at runtime
+                                 via an EFI_RT_PROPERTIES_TABLE configuration table.
 
 **/
 typedef
@@ -796,6 +828,9 @@ typedef struct {
   @retval EFI_SUCCESS           The operation completed successfully.
   @retval EFI_INVALID_PARAMETER Time is NULL.
   @retval EFI_DEVICE_ERROR      The time could not be retrieved due to hardware error.
+  @retval EFI_UNSUPPORTED       This call is not supported by this platform at the time the call is made.
+                                The platform should describe this runtime service as unsupported at runtime
+                                via an EFI_RT_PROPERTIES_TABLE configuration table.
 
 **/
 typedef
@@ -813,6 +848,9 @@ EFI_STATUS
   @retval EFI_SUCCESS           The operation completed successfully.
   @retval EFI_INVALID_PARAMETER A time field is out of range.
   @retval EFI_DEVICE_ERROR      The time could not be set due due to hardware error.
+  @retval EFI_UNSUPPORTED       This call is not supported by this platform at the time the call is made.
+                                The platform should describe this runtime service as unsupported at runtime
+                                via an EFI_RT_PROPERTIES_TABLE configuration table.
 
 **/
 typedef
@@ -833,7 +871,9 @@ EFI_STATUS
   @retval EFI_INVALID_PARAMETER Pending is NULL.
   @retval EFI_INVALID_PARAMETER Time is NULL.
   @retval EFI_DEVICE_ERROR      The wakeup time could not be retrieved due to a hardware error.
-  @retval EFI_UNSUPPORTED       A wakeup timer is not supported on this platform.
+  @retval EFI_UNSUPPORTED       This call is not supported by this platform at the time the call is made.
+                                The platform should describe this runtime service as unsupported at runtime
+                                via an EFI_RT_PROPERTIES_TABLE configuration table.
 
 **/
 typedef
@@ -855,7 +895,9 @@ EFI_STATUS
                                 Enable is FALSE, then the wakeup alarm was disabled.
   @retval EFI_INVALID_PARAMETER A time field is out of range.
   @retval EFI_DEVICE_ERROR      The wakeup time could not be set due to a hardware error.
-  @retval EFI_UNSUPPORTED       A wakeup timer is not supported on this platform.
+  @retval EFI_UNSUPPORTED       This call is not supported by this platform at the time the call is made.
+                                The platform should describe this runtime service as unsupported at runtime
+                                via an EFI_RT_PROPERTIES_TABLE configuration table.
 
 **/
 typedef
@@ -900,7 +942,7 @@ EFI_STATUS
 (EFIAPI *EFI_IMAGE_LOAD)(
   IN  BOOLEAN                      BootPolicy,
   IN  EFI_HANDLE                   ParentImageHandle,
-  IN  EFI_DEVICE_PATH_PROTOCOL     *DevicePath,
+  IN  EFI_DEVICE_PATH_PROTOCOL     *DevicePath   OPTIONAL,
   IN  VOID                         *SourceBuffer OPTIONAL,
   IN  UINTN                        SourceSize,
   OUT EFI_HANDLE                   *ImageHandle
@@ -1077,6 +1119,9 @@ EFI_STATUS
   @retval EFI_SUCCESS           The next high monotonic count was returned.
   @retval EFI_INVALID_PARAMETER HighCount is NULL.
   @retval EFI_DEVICE_ERROR      The device is not functioning properly.
+  @retval EFI_UNSUPPORTED       This call is not supported by this platform at the time the call is made.
+                                The platform should describe this runtime service as unsupported at runtime
+                                via an EFI_RT_PROPERTIES_TABLE configuration table.
 
 **/
 typedef
@@ -1650,7 +1695,7 @@ typedef struct {
   ///
   UINT32      Flags;
   ///
-  /// Size in bytes of the capsule.
+  /// Size in bytes of the capsule (including capsule header).
   ///
   UINT32      CapsuleImageSize;
 } EFI_CAPSULE_HEADER;
@@ -1703,6 +1748,9 @@ typedef struct {
                                 in runtime. The caller may resubmit the capsule prior to ExitBootServices().
   @retval EFI_OUT_OF_RESOURCES  When ExitBootServices() has not been previously called then this error indicates
                                 the capsule is compatible with this platform but there are insufficient resources to process.
+  @retval EFI_UNSUPPORTED       This call is not supported by this platform at the time the call is made.
+                                The platform should describe this runtime service as unsupported at runtime
+                                via an EFI_RT_PROPERTIES_TABLE configuration table.
 
 **/
 typedef
@@ -1734,6 +1782,9 @@ EFI_STATUS
                                 in runtime. The caller may resubmit the capsule prior to ExitBootServices().
   @retval EFI_OUT_OF_RESOURCES  When ExitBootServices() has not been previously called then this error indicates
                                 the capsule is compatible with this platform but there are insufficient resources to process.
+  @retval EFI_UNSUPPORTED       This call is not supported by this platform at the time the call is made.
+                                The platform should describe this runtime service as unsupported at runtime
+                                via an EFI_RT_PROPERTIES_TABLE configuration table.
 
 **/
 typedef
diff --git a/src/include/ipxe/efi/efi_path.h b/src/include/ipxe/efi/efi_path.h
index e75ae42c..20ff43f6 100644
--- a/src/include/ipxe/efi/efi_path.h
+++ b/src/include/ipxe/efi/efi_path.h
@@ -20,6 +20,7 @@ struct aoe_device;
 struct fcp_description;
 struct ib_srp_device;
 struct usb_function;
+union uuid;
 
 /**
  * Terminate device path
@@ -43,6 +44,7 @@ extern EFI_DEVICE_PATH_PROTOCOL *
 efi_path_end ( EFI_DEVICE_PATH_PROTOCOL *path );
 extern size_t efi_path_len ( EFI_DEVICE_PATH_PROTOCOL *path );
 extern unsigned int efi_path_vlan ( EFI_DEVICE_PATH_PROTOCOL *path );
+extern int efi_path_guid ( EFI_DEVICE_PATH_PROTOCOL *path, union uuid *uuid );
 extern EFI_DEVICE_PATH_PROTOCOL * efi_paths ( EFI_DEVICE_PATH_PROTOCOL *first,
 					      ... );
 extern EFI_DEVICE_PATH_PROTOCOL * efi_netdev_path ( struct net_device *netdev );
diff --git a/src/include/ipxe/efi/import.pl b/src/include/ipxe/efi/import.pl
index 34aed9a2..0a7669f4 100755
--- a/src/include/ipxe/efi/import.pl
+++ b/src/include/ipxe/efi/import.pl
@@ -68,7 +68,7 @@ sub try_import_file {
 	chomp;
 	# Update include lines, and record included files
 	if ( s/^(\s*\#include\s+)[<\"](\S+)[>\"]/$1<ipxe\/efi\/$2>/ ) {
-	  push @dependencies, $1;
+	  push @dependencies, $2;
 	}
 	# Check for BSD licence statement
 	if ( /^\s*SPDX-License-Identifier: BSD-2-Clause-Patent$/ ) {
diff --git a/src/include/ipxe/entropy.h b/src/include/ipxe/entropy.h
index 240feace..82bb1182 100644
--- a/src/include/ipxe/entropy.h
+++ b/src/include/ipxe/entropy.h
@@ -237,8 +237,7 @@ get_entropy_input ( unsigned int min_entropy_bits, void *data, size_t min_len,
 	int rc;
 
 	/* Sanity check */
-	linker_assert ( ( min_entropy_bits <= ( 8 * max_len ) ),
-			entropy_buffer_too_small );
+	build_assert ( min_entropy_bits <= ( 8 * max_len ) );
 
 	/* Round up minimum entropy to an integral number of bytes */
 	min_entropy_bits = ( ( min_entropy_bits + 7 ) & ~7 );
@@ -247,11 +246,11 @@ get_entropy_input ( unsigned int min_entropy_bits, void *data, size_t min_len,
 	 * meet or exceed the security strength indicated by the
 	 * min_entropy parameter.
 	 */
-	linker_assert ( ( ( 8 * ENTROPY_HASH_DF_OUTLEN_BYTES ) >=
-			  min_entropy_bits ), hash_df_algorithm_too_weak );
+	build_assert ( ( 8 * ENTROPY_HASH_DF_OUTLEN_BYTES ) >=
+		       min_entropy_bits );
 
 	/* 1.  If ( min_length > max_length ), then return ( FAILURE, Null ) */
-	linker_assert ( ( min_len <= max_len ), min_len_greater_than_max_len );
+	build_assert ( min_len <= max_len );
 
 	/* 2.  n = 2 * min_entropy */
 	n = ( 2 * min_entropy_bits );
@@ -269,9 +268,8 @@ get_entropy_input ( unsigned int min_entropy_bits, void *data, size_t min_len,
 	 * (The implementation of these steps is inside the function
 	 * get_entropy_input_tmp().)
 	 */
-	linker_assert ( __builtin_constant_p ( tmp_len ),
-			tmp_len_not_constant );
-	linker_assert ( ( n == ( 8 * tmp_len ) ), tmp_len_mismatch );
+	build_assert ( __builtin_constant_p ( tmp_len ) );
+	build_assert ( n == ( 8 * tmp_len ) );
 	if ( ( rc = get_entropy_input_tmp ( MIN_ENTROPY ( min_entropy_bits ),
 					    tmp, tmp_len ) ) != 0 ) {
 		return rc;
@@ -283,17 +281,17 @@ get_entropy_input ( unsigned int min_entropy_bits, void *data, size_t min_len,
 	 */
 	if ( tmp_len < min_len ) {
 		/* (Data is already in-place.) */
-		linker_assert ( ( data == tmp ), data_not_inplace );
+		build_assert ( data == tmp );
 		memset ( ( data + tmp_len ), 0, ( min_len - tmp_len ) );
 		return min_len;
 	} else if ( tmp_len > max_len ) {
-		linker_assert ( ( tmp == tmp_buf ), data_inplace );
+		build_assert ( tmp == tmp_buf );
 		hash_df ( &entropy_hash_df_algorithm, tmp, tmp_len,
 			  data, max_len );
 		return max_len;
 	} else {
 		/* (Data is already in-place.) */
-		linker_assert ( ( data == tmp ), data_not_inplace );
+		build_assert ( data == tmp );
 		return tmp_len;
 	}
 }
@@ -328,15 +326,14 @@ entropy_repetition_count_cutoff ( min_entropy_t min_entropy_per_sample ) {
 	cutoff = max_repetitions;
 	if ( cutoff < max_repetitions )
 		cutoff++;
-	linker_assert ( ( cutoff >= max_repetitions ), rounding_error );
+	build_assert ( cutoff >= max_repetitions );
 
 	/* Floating-point operations are not allowed in iPXE since we
 	 * never set up a suitable environment.  Abort the build
 	 * unless the calculated number of repetitions is a
 	 * compile-time constant.
 	 */
-	linker_assert ( __builtin_constant_p ( cutoff ),
-			repetition_count_cutoff_not_constant );
+	build_assert ( __builtin_constant_p ( cutoff ) );
 
 	return cutoff;
 }
@@ -443,12 +440,10 @@ entropy_adaptive_proportion_cutoff ( min_entropy_t min_entropy_per_sample ) {
 	cutoff = entropy_adaptive_proportion_cutoff_lookup ( n, h );
 
 	/* Fail unless cutoff value is a compile-time constant */
-	linker_assert ( __builtin_constant_p ( cutoff ),
-			adaptive_proportion_cutoff_not_constant );
+	build_assert ( __builtin_constant_p ( cutoff ) );
 
 	/* Fail if cutoff value is N/A */
-	linker_assert ( ( cutoff != APC_NA ),
-			adaptive_proportion_cutoff_not_applicable );
+	build_assert ( cutoff != APC_NA );
 
 	return cutoff;
 }
@@ -475,8 +470,7 @@ entropy_startup_test_count ( unsigned int repetition_count_cutoff,
 	num_samples = repetition_count_cutoff;
 	if ( num_samples < adaptive_proportion_cutoff )
 		num_samples = adaptive_proportion_cutoff;
-	linker_assert ( __builtin_constant_p ( num_samples ),
-			startup_test_count_not_constant );
+	build_assert ( __builtin_constant_p ( num_samples ) );
 
 	return num_samples;
 }
@@ -499,11 +493,9 @@ entropy_init ( struct entropy_source *source,
 	unsigned int startup_test_count;
 
 	/* Sanity check */
-	linker_assert ( min_entropy_per_sample > MIN_ENTROPY ( 0 ),
-			min_entropy_per_sample_is_zero );
-	linker_assert ( ( min_entropy_per_sample <=
-			  MIN_ENTROPY ( 8 * sizeof ( noise_sample_t ) ) ),
-			min_entropy_per_sample_is_impossibly_high );
+	build_assert ( min_entropy_per_sample > MIN_ENTROPY ( 0 ) );
+	build_assert ( min_entropy_per_sample <=
+		       MIN_ENTROPY ( 8 * sizeof ( noise_sample_t ) ) );
 
 	/* Calculate test cutoff values */
 	repetition_count_cutoff =
diff --git a/src/include/ipxe/errfile.h b/src/include/ipxe/errfile.h
index 320835a3..21c3d338 100644
--- a/src/include/ipxe/errfile.h
+++ b/src/include/ipxe/errfile.h
@@ -79,6 +79,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_cachedhcp	       ( ERRFILE_CORE | 0x00270000 )
 #define ERRFILE_acpimac		       ( ERRFILE_CORE | 0x00280000 )
 #define ERRFILE_efi_strings	       ( ERRFILE_CORE | 0x00290000 )
+#define ERRFILE_uuid		       ( ERRFILE_CORE | 0x002a0000 )
+#define ERRFILE_efi_path	       ( ERRFILE_CORE | 0x002b0000 )
 
 #define ERRFILE_eisa		     ( ERRFILE_DRIVER | 0x00000000 )
 #define ERRFILE_isa		     ( ERRFILE_DRIVER | 0x00010000 )
@@ -219,6 +221,10 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_ice		     ( ERRFILE_DRIVER | 0x00d20000 )
 #define ERRFILE_ecam		     ( ERRFILE_DRIVER | 0x00d30000 )
 #define ERRFILE_pcibridge	     ( ERRFILE_DRIVER | 0x00d40000 )
+#define ERRFILE_aqc1xx		     ( ERRFILE_DRIVER | 0x00d50000 )
+#define ERRFILE_atl_hw		     ( ERRFILE_DRIVER | 0x00d60000 )
+#define ERRFILE_atl2_hw		     ( ERRFILE_DRIVER | 0x00d70000 )
+
 
 #define ERRFILE_aoe			( ERRFILE_NET | 0x00000000 )
 #define ERRFILE_arp			( ERRFILE_NET | 0x00010000 )
@@ -297,6 +303,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_httpntlm		( ERRFILE_NET | 0x004a0000 )
 #define ERRFILE_eap			( ERRFILE_NET | 0x004b0000 )
 #define ERRFILE_lldp			( ERRFILE_NET | 0x004c0000 )
+#define ERRFILE_eap_md5			( ERRFILE_NET | 0x004d0000 )
+#define ERRFILE_eap_mschapv2		( ERRFILE_NET | 0x004e0000 )
 
 #define ERRFILE_image		      ( ERRFILE_IMAGE | 0x00000000 )
 #define ERRFILE_elf		      ( ERRFILE_IMAGE | 0x00010000 )
@@ -407,6 +415,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_efi_rng		      ( ERRFILE_OTHER | 0x005c0000 )
 #define ERRFILE_efi_shim	      ( ERRFILE_OTHER | 0x005d0000 )
 #define ERRFILE_efi_settings	      ( ERRFILE_OTHER | 0x005e0000 )
+#define ERRFILE_x25519		      ( ERRFILE_OTHER | 0x005f0000 )
+#define ERRFILE_des		      ( ERRFILE_OTHER | 0x00600000 )
 
 /** @} */
 
diff --git a/src/include/ipxe/gcm.h b/src/include/ipxe/gcm.h
index 90ef0b52..4864445d 100644
--- a/src/include/ipxe/gcm.h
+++ b/src/include/ipxe/gcm.h
@@ -88,13 +88,11 @@ struct _gcm_name ## _context {						\
 static int _gcm_name ## _setkey ( void *ctx, const void *key,		\
 				  size_t keylen ) {			\
 	struct _gcm_name ## _context *context = ctx;			\
-	linker_assert ( _blocksize == sizeof ( context->gcm.key ),	\
-			_gcm_name ## _unsupported_blocksize );		\
-	linker_assert ( ( ( void * ) &context->gcm ) == ctx,		\
-			_gcm_name ## _context_layout_error );		\
-	linker_assert ( ( ( void * ) &context->raw ) ==			\
-			( ( void * ) context->gcm.raw_ctx ),		\
-			_gcm_name ## _context_layout_error );		\
+	build_assert ( _blocksize == sizeof ( context->gcm.key ) );	\
+	build_assert ( ( ( void * ) &context->gcm ) ==			\
+		       ( ( void * ) context ) );			\
+	build_assert ( ( ( void * ) &context->raw ) ==			\
+		       ( ( void * ) context->gcm.raw_ctx ) );		\
 	return gcm_setkey ( &context->gcm, key, keylen, &_raw_cipher );	\
 }									\
 static void _gcm_name ## _setiv ( void *ctx, const void *iv,		\
diff --git a/src/include/ipxe/list.h b/src/include/ipxe/list.h
index 8de25498..2f02e71f 100644
--- a/src/include/ipxe/list.h
+++ b/src/include/ipxe/list.h
@@ -399,6 +399,17 @@ extern void extern_list_splice_tail_init ( struct list_head *list,
 	( (head)->prev == &(entry)->member )
 
 /**
+ * Test if entry is the list head
+ *
+ * @v entry		List entry
+ * @v head		List head
+ * @v member		Name of list field within iterator's type
+ * @ret is_head		Entry is the list head
+ */
+#define list_is_head_entry( entry, head, member )		\
+	( (head) == &(entry)->member )
+
+/**
  * Iterate over a list
  *
  * @v pos		Iterator
@@ -479,6 +490,22 @@ extern void extern_list_splice_tail_init ( struct list_head *list,
 	      pos = list_entry ( pos->member.prev, typeof ( *pos ), member ) )
 
 /**
+ * Iterate over subsequent entries in a list, safe against deletion
+ *
+ * @v pos		Iterator
+ * @v tmp		Temporary value (of same type as iterator)
+ * @v head		List head
+ * @v member		Name of list field within iterator's type
+ */
+#define list_for_each_entry_safe_continue( pos, tmp, head, member )	      \
+	for ( list_check ( (head) ),					      \
+	      pos = list_entry ( pos->member.next, typeof ( *pos ), member ), \
+	      tmp = list_entry ( pos->member.next, typeof ( *tmp ), member ); \
+	      &pos->member != (head);					      \
+	      pos = tmp,						      \
+	      tmp = list_entry ( tmp->member.next, typeof ( *tmp ), member ) )
+
+/**
  * Test if list contains a specified entry
  *
  * @v entry		Entry
diff --git a/src/include/ipxe/mschapv2.h b/src/include/ipxe/mschapv2.h
new file mode 100644
index 00000000..59cf37ee
--- /dev/null
+++ b/src/include/ipxe/mschapv2.h
@@ -0,0 +1,59 @@
+#ifndef _IPXE_MSCHAPV2_H
+#define _IPXE_MSCHAPV2_H
+
+/** @file
+ *
+ * MS-CHAPv2 authentication
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+
+/** An MS-CHAPv2 challenge */
+struct mschapv2_challenge {
+	/** Raw bytes */
+	uint8_t byte[16];
+} __attribute__ (( packed ));
+
+/** An MS-CHAPv2 NT response */
+struct mschapv2_nt_response {
+	/** DES-encrypted blocks */
+	uint8_t block[3][8];
+} __attribute__ (( packed ));
+
+/** An MS-CHAPv2 challenge response */
+struct mschapv2_response {
+	/** Peer challenge */
+	struct mschapv2_challenge peer;
+	/** Reserved, must be zero */
+	uint8_t reserved[8];
+	/** NT response */
+	struct mschapv2_nt_response nt;
+	/** Flags, must be zero */
+	uint8_t flags;
+} __attribute__ (( packed ));
+
+/** An MS-CHAPv2 authenticator response */
+struct mschapv2_auth {
+	/** Authenticator response string
+	 *
+	 * This is an unterminated 42-byte string of the form
+	 * "S=<auth_string>" where <auth_string> is the upper-cased
+	 * hexadecimal encoding of the actual authenticator response
+	 * value.  Joy.
+	 */
+	char wtf[42];
+} __attribute__ (( packed ));
+
+extern void mschapv2_response ( const char *username, const char *password,
+				const struct mschapv2_challenge *challenge,
+				const struct mschapv2_challenge *peer,
+				struct mschapv2_response *response );
+extern void mschapv2_auth ( const char *username, const char *password,
+			    const struct mschapv2_challenge *challenge,
+			    const struct mschapv2_response *response,
+			    struct mschapv2_auth *auth );
+
+#endif /* _IPXE_MSCHAPV2_H */
diff --git a/src/include/ipxe/parseopt.h b/src/include/ipxe/parseopt.h
index 829b3431..61010f22 100644
--- a/src/include/ipxe/parseopt.h
+++ b/src/include/ipxe/parseopt.h
@@ -11,6 +11,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stddef.h>
+#include <ipxe/uuid.h>
 #include <ipxe/settings.h>
 
 struct net_device;
@@ -125,9 +126,18 @@ struct named_setting {
 	struct setting setting;
 };
 
+/** A UUID command-line option */
+struct uuid_option {
+	/** UUID */
+	union uuid *value;
+	/** Storage buffer */
+	union uuid buf;
+};
+
 extern int parse_string ( char *text, char **value );
 extern int parse_integer ( char *text, unsigned int *value );
 extern int parse_timeout ( char *text, unsigned long *value );
+extern int parse_uuid ( char *text, struct uuid_option *uuid );
 extern int parse_netdev ( char *text, struct net_device **netdev );
 extern int
 parse_netdev_configurator ( char *text,
diff --git a/src/include/ipxe/sanboot.h b/src/include/ipxe/sanboot.h
index b163a94b..e44367cd 100644
--- a/src/include/ipxe/sanboot.h
+++ b/src/include/ipxe/sanboot.h
@@ -19,8 +19,19 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/process.h>
 #include <ipxe/blockdev.h>
 #include <ipxe/acpi.h>
+#include <ipxe/uuid.h>
 #include <config/sanboot.h>
 
+/**
+ * Default SAN drive number
+ *
+ * The drive number is an externally defined concept only in a BIOS
+ * environment, where it represents the INT13 drive number (0x80 for
+ * the first hard disk).  We retain it in other environments to allow
+ * for a simple way for iPXE commands to refer to SAN drives.
+ */
+#define SAN_DEFAULT_DRIVE 0x80
+
 /** A SAN path */
 struct san_path {
 	/** Containing SAN device */
@@ -95,6 +106,18 @@ enum san_device_flags {
 	SAN_NO_DESCRIBE = 0x0001,
 };
 
+/** SAN boot configuration parameters */
+struct san_boot_config {
+	/** Boot filename (or NULL to use default) */
+	const char *filename;
+	/** Required extra filename (or NULL to ignore) */
+	const char *extra;
+	/** Filesystem label (or NULL to ignore volume label) */
+	const char *label;
+	/** UUID (or NULL to ignore UUID) */
+	union uuid *uuid;
+};
+
 /**
  * Calculate static inline sanboot API function name
  *
@@ -155,10 +178,10 @@ void san_unhook ( unsigned int drive );
  * Attempt to boot from a SAN device
  *
  * @v drive		Drive number
- * @v filename		Filename (or NULL to use default)
+ * @v config		Boot configuration parameters
  * @ret rc		Return status code
  */
-int san_boot ( unsigned int drive, const char *filename );
+int san_boot ( unsigned int drive, struct san_boot_config *config );
 
 /**
  * Describe SAN devices for SAN-booted operating system
@@ -234,6 +257,7 @@ static inline int sandev_needs_reopen ( struct san_device *sandev ) {
 }
 
 extern struct san_device * sandev_find ( unsigned int drive );
+extern struct san_device * sandev_next ( unsigned int drive );
 extern int sandev_reopen ( struct san_device *sandev );
 extern int sandev_reset ( struct san_device *sandev );
 extern int sandev_read ( struct san_device *sandev, uint64_t lba,
diff --git a/src/include/ipxe/smbios.h b/src/include/ipxe/smbios.h
index 42278fb2..077a67a8 100644
--- a/src/include/ipxe/smbios.h
+++ b/src/include/ipxe/smbios.h
@@ -227,6 +227,8 @@ struct smbios {
 extern int find_smbios ( struct smbios *smbios );
 extern int find_smbios_entry ( userptr_t start, size_t len,
 			       struct smbios_entry *entry );
+extern int find_smbios3_entry ( userptr_t start, size_t len,
+				struct smbios3_entry *entry );
 extern int find_smbios_structure ( unsigned int type, unsigned int instance,
 				   struct smbios_structure *structure );
 extern int read_smbios_structure ( struct smbios_structure *structure,
diff --git a/src/include/ipxe/tls.h b/src/include/ipxe/tls.h
index 30bb1c48..cf327782 100644
--- a/src/include/ipxe/tls.h
+++ b/src/include/ipxe/tls.h
@@ -96,6 +96,12 @@ struct tls_header {
 #define TLS_RSA_WITH_AES_256_GCM_SHA384 0x009d
 #define TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 0x009e
 #define TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 0x009f
+#define TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA 0xc013
+#define TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA 0xc014
+#define TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 0xc027
+#define TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 0xc028
+#define TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 0xc02f
+#define TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 0xc030
 
 /* TLS hash algorithm identifiers */
 #define TLS_MD5_ALGORITHM 1
@@ -119,6 +125,10 @@ struct tls_header {
 #define TLS_MAX_FRAGMENT_LENGTH_2048 3
 #define TLS_MAX_FRAGMENT_LENGTH_4096 4
 
+/* TLS named curve extension */
+#define TLS_NAMED_CURVE 10
+#define TLS_NAMED_CURVE_X25519 29
+
 /* TLS signature algorithms extension */
 #define TLS_SIGNATURE_ALGORITHMS 13
 
@@ -205,6 +215,25 @@ struct tls_cipher_suite {
 #define __tls_cipher_suite( pref )					\
 	__table_entry ( TLS_CIPHER_SUITES, pref )
 
+/** TLS named curved type */
+#define TLS_NAMED_CURVE_TYPE 3
+
+/** A TLS named curve */
+struct tls_named_curve {
+	/** Elliptic curve */
+	struct elliptic_curve *curve;
+	/** Numeric code (in network-endian order) */
+	uint16_t code;
+};
+
+/** TLS named curve table */
+#define TLS_NAMED_CURVES						\
+	__table ( struct tls_named_curve, "tls_named_curves" )
+
+/** Declare a TLS named curve */
+#define __tls_named_curve( pref )					\
+	__table_entry ( TLS_NAMED_CURVES, pref )
+
 /** A TLS cipher specification */
 struct tls_cipherspec {
 	/** Cipher suite */
@@ -425,6 +454,7 @@ struct tls_connection {
 
 extern struct tls_key_exchange_algorithm tls_pubkey_exchange_algorithm;
 extern struct tls_key_exchange_algorithm tls_dhe_exchange_algorithm;
+extern struct tls_key_exchange_algorithm tls_ecdhe_exchange_algorithm;
 
 extern int add_tls ( struct interface *xfer, const char *name,
 		     struct x509_root *root, struct private_key *key );
diff --git a/src/include/ipxe/uuid.h b/src/include/ipxe/uuid.h
index 24c46aca..4874b738 100644
--- a/src/include/ipxe/uuid.h
+++ b/src/include/ipxe/uuid.h
@@ -48,5 +48,6 @@ static inline void uuid_mangle ( union uuid *uuid ) {
 }
 
 extern const char * uuid_ntoa ( const union uuid *uuid );
+extern int uuid_aton ( const char *string, union uuid *uuid );
 
 #endif /* _IPXE_UUID_H */
diff --git a/src/include/ipxe/x25519.h b/src/include/ipxe/x25519.h
new file mode 100644
index 00000000..fd7caeee
--- /dev/null
+++ b/src/include/ipxe/x25519.h
@@ -0,0 +1,94 @@
+#ifndef _IPXE_X25519_H
+#define _IPXE_X25519_H
+
+/** @file
+ *
+ * X25519 key exchange
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/bigint.h>
+#include <ipxe/crypto.h>
+
+/** X25519 unsigned big integer size
+ *
+ * X25519 uses the finite field of integers modulo the prime
+ * p=2^255-19.  The canonical representations of integers in this
+ * field therefore require only 255 bits.
+ *
+ * For internal calculations we use big integers containing up to 267
+ * bits, since this ends up allowing us to avoid some unnecessary (and
+ * expensive) intermediate reductions modulo p.
+ */
+#define X25519_SIZE bigint_required_size ( ( 267 /* bits */ + 7 ) / 8 )
+
+/** An X25519 unsigned big integer used in internal calculations */
+typedef bigint_t ( X25519_SIZE ) x25519_t;
+
+/** An X25519 unsigned 258-bit integer
+ *
+ * This is an unsigned integer N in the finite field of integers
+ * modulo the prime p=2^255-19.
+ *
+ * In this representation, N is encoded as any big integer that is in
+ * the same congruence class as N (i.e that has the same value as N
+ * modulo p) and that lies within the 258-bit range [0,8p-1].
+ *
+ * This type can be used as an input for multiplication (but not for
+ * addition or subtraction).
+ *
+ * Addition or subtraction will produce an output of this type.
+ */
+union x25519_oct258 {
+	/** Big integer value */
+	x25519_t value;
+};
+
+/** An X25519 unsigned 257-bit integer
+ *
+ * This is an unsigned integer N in the finite field of integers
+ * modulo the prime p=2^255-19.
+ *
+ * In this representation, N is encoded as any big integer that is in
+ * the same congruence class as N (i.e that has the same value as N
+ * modulo p) and that lies within the 257-bit range [0,4p-1].
+ *
+ * This type can be used as an input for addition, subtraction, or
+ * multiplication.
+ *
+ * Multiplication will produce an output of this type.
+ */
+union x25519_quad257 {
+	/** Big integer value */
+	x25519_t value;
+	/** X25519 unsigned 258-bit integer
+	 *
+	 * Any value in the range [0,4p-1] is automatically also
+	 * within the range [0,8p-1] and so may be consumed as an
+	 * unsigned 258-bit integer.
+	 */
+	const union x25519_oct258 oct258;
+};
+
+/** An X25519 32-byte value */
+struct x25519_value {
+	/** Raw value */
+	uint8_t raw[32];
+};
+
+extern void x25519_multiply ( const union x25519_oct258 *multiplicand,
+			      const union x25519_oct258 *multiplier,
+			      union x25519_quad257 *result );
+extern void x25519_invert ( const union x25519_oct258 *invertend,
+			    union x25519_quad257 *result );
+extern void x25519_reduce ( union x25519_quad257 *value );
+extern int x25519_key ( const struct x25519_value *base,
+			const struct x25519_value *scalar,
+			struct x25519_value *result );
+
+extern struct elliptic_curve x25519_curve;
+
+#endif /* _IPXE_X25519_H */
diff --git a/src/include/ipxe/x509.h b/src/include/ipxe/x509.h
index c703c8f1..87323cec 100644
--- a/src/include/ipxe/x509.h
+++ b/src/include/ipxe/x509.h
@@ -171,6 +171,28 @@ struct x509_link {
 	struct list_head list;
 	/** Certificate */
 	struct x509_certificate *cert;
+	/** Flags */
+	unsigned int flags;
+};
+
+/** X.509 certficate chain link flags */
+enum x509_link_flags {
+	/** Cross-signed certificate download has been attempted
+	 *
+	 * This indicates that a cross-signature download attempt has
+	 * been made to find a cross-signed issuer for this link's
+	 * certificate.
+	 */
+	X509_LINK_FL_CROSSED = 0x0001,
+	/** OCSP has been attempted
+	 *
+	 * This indicates that an OCSP attempt has been made using
+	 * this link's certificate as an issuer.  (We record the flag
+	 * on the issuer rather than on the issued certificate, since
+	 * we want to retry OCSP if an issuer is replaced with a
+	 * downloaded cross-signed certificate.)
+	 */
+	X509_LINK_FL_OCSPED = 0x0002,
 };
 
 /** An X.509 certificate chain */
@@ -374,6 +396,16 @@ x509_root_put ( struct x509_root *root ) {
 	ref_put ( &root->refcnt );
 }
 
+/**
+ * Check if X.509 certificate is self-signed
+ *
+ * @v cert		X.509 certificate
+ * @ret is_self_signed	X.509 certificate is self-signed
+ */
+static inline int x509_is_self_signed ( struct x509_certificate *cert ) {
+	return ( asn1_compare ( &cert->issuer.raw, &cert->subject.raw ) == 0 );
+}
+
 extern const char * x509_name ( struct x509_certificate *cert );
 extern int x509_parse ( struct x509_certificate *cert,
 			const struct asn1_cursor *raw );
@@ -391,6 +423,7 @@ extern int x509_append ( struct x509_chain *chain,
 			 struct x509_certificate *cert );
 extern int x509_append_raw ( struct x509_chain *chain, const void *data,
 			     size_t len );
+extern void x509_truncate ( struct x509_chain *chain, struct x509_link *link );
 extern int x509_auto_append ( struct x509_chain *chain,
 			      struct x509_chain *certs );
 extern int x509_validate_chain ( struct x509_chain *chain, time_t time,
diff --git a/src/include/usr/autoboot.h b/src/include/usr/autoboot.h
index 3719b824..a081a70d 100644
--- a/src/include/usr/autoboot.h
+++ b/src/include/usr/autoboot.h
@@ -14,6 +14,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 struct net_device;
 struct uri;
 struct settings;
+struct san_boot_config;
 
 /** uriboot() flags */
 enum uriboot_flags {
@@ -33,7 +34,7 @@ extern void set_autoboot_ll_addr ( const void *ll_addr, size_t len,
 
 extern int uriboot ( struct uri *filename, struct uri **root_paths,
 		     unsigned int root_path_count, int drive,
-		     const char *san_filename, unsigned int flags );
+		     struct san_boot_config *san_config, unsigned int flags );
 extern struct uri *
 fetch_next_server_and_filename ( struct settings *settings );
 extern int netboot ( struct net_device *netdev );
diff --git a/src/interface/efi/efi_block.c b/src/interface/efi/efi_block.c
index da906210..a9c3d656 100644
--- a/src/interface/efi/efi_block.c
+++ b/src/interface/efi/efi_block.c
@@ -32,7 +32,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdlib.h>
+#include <stdio.h>
 #include <string.h>
+#include <strings.h>
 #include <errno.h>
 #include <ipxe/refcnt.h>
 #include <ipxe/list.h>
@@ -51,6 +53,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/efi/Protocol/BlockIo.h>
 #include <ipxe/efi/Protocol/SimpleFileSystem.h>
 #include <ipxe/efi/Protocol/AcpiTable.h>
+#include <ipxe/efi/Guid/FileSystemInfo.h>
 #include <ipxe/efi/efi_driver.h>
 #include <ipxe/efi/efi_strings.h>
 #include <ipxe/efi/efi_snp.h>
@@ -102,7 +105,7 @@ static int efi_block_rw ( struct san_device *sandev, uint64_t lba,
 	/* Sanity check */
 	count = ( len / block->media.BlockSize );
 	if ( ( count * block->media.BlockSize ) != len ) {
-		DBGC ( sandev, "EFIBLK %#02x impossible length %#zx\n",
+		DBGC ( sandev->drive, "EFIBLK %#02x impossible length %#zx\n",
 		       sandev->drive, len );
 		return -EINVAL;
 	}
@@ -110,7 +113,7 @@ static int efi_block_rw ( struct san_device *sandev, uint64_t lba,
 	/* Read from / write to block device */
 	if ( ( rc = sandev_rw ( sandev, lba, count,
 				virt_to_user ( data ) ) ) != 0 ) {
-		DBGC ( sandev, "EFIBLK %#02x I/O failed: %s\n",
+		DBGC ( sandev->drive, "EFIBLK %#02x I/O failed: %s\n",
 		       sandev->drive, strerror ( rc ) );
 		return rc;
 	}
@@ -132,7 +135,7 @@ static EFI_STATUS EFIAPI efi_block_io_reset ( EFI_BLOCK_IO_PROTOCOL *block_io,
 	struct san_device *sandev = block->sandev;
 	int rc;
 
-	DBGC2 ( sandev, "EFIBLK %#02x reset\n", sandev->drive );
+	DBGC2 ( sandev->drive, "EFIBLK %#02x reset\n", sandev->drive );
 	efi_snp_claim();
 	rc = sandev_reset ( sandev );
 	efi_snp_release();
@@ -157,7 +160,7 @@ efi_block_io_read ( EFI_BLOCK_IO_PROTOCOL *block_io, UINT32 media __unused,
 	struct san_device *sandev = block->sandev;
 	int rc;
 
-	DBGC2 ( sandev, "EFIBLK %#02x read LBA %#08llx to %p+%#08zx\n",
+	DBGC2 ( sandev->drive, "EFIBLK %#02x read LBA %#08llx to %p+%#08zx\n",
 		sandev->drive, lba, data, ( ( size_t ) len ) );
 	efi_snp_claim();
 	rc = efi_block_rw ( sandev, lba, data, len, sandev_read );
@@ -183,8 +186,8 @@ efi_block_io_write ( EFI_BLOCK_IO_PROTOCOL *block_io, UINT32 media __unused,
 	struct san_device *sandev = block->sandev;
 	int rc;
 
-	DBGC2 ( sandev, "EFIBLK %#02x write LBA %#08llx from %p+%#08zx\n",
-		sandev->drive, lba, data, ( ( size_t ) len ) );
+	DBGC2 ( sandev->drive, "EFIBLK %#02x write LBA %#08llx from "
+		"%p+%#08zx\n", sandev->drive, lba, data, ( ( size_t ) len ) );
 	efi_snp_claim();
 	rc = efi_block_rw ( sandev, lba, data, len, sandev_write );
 	efi_snp_release();
@@ -203,7 +206,7 @@ efi_block_io_flush ( EFI_BLOCK_IO_PROTOCOL *block_io ) {
 		container_of ( block_io, struct efi_block_data, block_io );
 	struct san_device *sandev = block->sandev;
 
-	DBGC2 ( sandev, "EFIBLK %#02x flush\n", sandev->drive );
+	DBGC2 ( sandev->drive, "EFIBLK %#02x flush\n", sandev->drive );
 
 	/* Nothing to do */
 	return 0;
@@ -212,24 +215,24 @@ efi_block_io_flush ( EFI_BLOCK_IO_PROTOCOL *block_io ) {
 /**
  * Connect all possible drivers to EFI block device
  *
- * @v sandev		SAN device
+ * @v drive		Drive number
+ * @v handle		Block device handle
  */
-static void efi_block_connect ( struct san_device *sandev ) {
+static void efi_block_connect ( unsigned int drive, EFI_HANDLE handle ) {
 	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
-	struct efi_block_data *block = sandev->priv;
 	EFI_STATUS efirc;
 	int rc;
 
 	/* Try to connect all possible drivers to this block device */
-	if ( ( efirc = bs->ConnectController ( block->handle, NULL,
-					       NULL, TRUE ) ) != 0 ) {
+	if ( ( efirc = bs->ConnectController ( handle, NULL, NULL,
+					       TRUE ) ) != 0 ) {
 		rc = -EEFI ( efirc );
-		DBGC ( sandev, "EFIBLK %#02x could not connect drivers: %s\n",
-		       sandev->drive, strerror ( rc ) );
+		DBGC ( drive, "EFIBLK %#02x could not connect drivers: %s\n",
+		       drive, strerror ( rc ) );
 		/* May not be an error; may already be connected */
 	}
-	DBGC2 ( sandev, "EFIBLK %#02x supports protocols:\n", sandev->drive );
-	DBGC2_EFI_PROTOCOLS ( sandev, block->handle );
+	DBGC2 ( drive, "EFIBLK %#02x supports protocols:\n", drive );
+	DBGC2_EFI_PROTOCOLS ( drive, handle );
 }
 
 /**
@@ -252,7 +255,7 @@ static int efi_block_hook ( unsigned int drive, struct uri **uris,
 
 	/* Sanity check */
 	if ( ! count ) {
-		DBG ( "EFIBLK has no URIs\n" );
+		DBGC ( drive, "EFIBLK %#02x has no URIs\n", drive );
 		rc = -ENOTTY;
 		goto err_no_uris;
 	}
@@ -276,7 +279,7 @@ static int efi_block_hook ( unsigned int drive, struct uri **uris,
 
 	/* Register SAN device */
 	if ( ( rc = register_sandev ( sandev, drive, flags ) ) != 0 ) {
-		DBGC ( sandev, "EFIBLK %#02x could not register: %s\n",
+		DBGC ( drive, "EFIBLK %#02x could not register: %s\n",
 		       drive, strerror ( rc ) );
 		goto err_register;
 	}
@@ -290,18 +293,18 @@ static int efi_block_hook ( unsigned int drive, struct uri **uris,
 	/* Construct device path */
 	if ( ! sandev->active ) {
 		rc = -ENODEV;
-		DBGC ( sandev, "EFIBLK %#02x not active after registration\n",
+		DBGC ( drive, "EFIBLK %#02x not active after registration\n",
 		       drive );
 		goto err_active;
 	}
 	block->path = efi_describe ( &sandev->active->block );
 	if ( ! block->path ) {
 		rc = -ENODEV;
-		DBGC ( sandev, "EFIBLK %#02x has no device path\n", drive );
+		DBGC ( drive, "EFIBLK %#02x has no device path\n", drive );
 		goto err_describe;
 	}
-	DBGC ( sandev, "EFIBLK %#02x has device path %s\n",
-	       drive, efi_devpath_text ( block->path ) );
+	DBGC2 ( drive, "EFIBLK %#02x has device path %s\n",
+		drive, efi_devpath_text ( block->path ) );
 
 	/* Install protocols */
 	if ( ( efirc = bs->InstallMultipleProtocolInterfaces (
@@ -310,13 +313,15 @@ static int efi_block_hook ( unsigned int drive, struct uri **uris,
 			&efi_device_path_protocol_guid, block->path,
 			NULL ) ) != 0 ) {
 		rc = -EEFI ( efirc );
-		DBGC ( sandev, "EFIBLK %#02x could not install protocols: %s\n",
-		       sandev->drive, strerror ( rc ) );
+		DBGC ( drive, "EFIBLK %#02x could not install protocols: %s\n",
+		       drive, strerror ( rc ) );
 		goto err_install;
 	}
+	DBGC ( drive, "EFIBLK %#02x installed as SAN drive %s\n",
+	       drive, efi_handle_name ( block->handle ) );
 
 	/* Connect all possible protocols */
-	efi_block_connect ( sandev );
+	efi_block_connect ( drive, block->handle );
 
 	return drive;
 
@@ -325,8 +330,8 @@ static int efi_block_hook ( unsigned int drive, struct uri **uris,
 			&efi_block_io_protocol_guid, &block->block_io,
 			&efi_device_path_protocol_guid, block->path,
 			NULL ) ) != 0 ) {
-		DBGC ( sandev, "EFIBLK %#02x could not uninstall protocols: "
-		       "%s\n", sandev->drive, strerror ( -EEFI ( efirc ) ) );
+		DBGC ( drive, "EFIBLK %#02x could not uninstall protocols: "
+		       "%s\n", drive, strerror ( -EEFI ( efirc ) ) );
 		leak = 1;
 	}
 	efi_nullify_block ( &block->block_io );
@@ -343,10 +348,8 @@ static int efi_block_hook ( unsigned int drive, struct uri **uris,
 		sandev_put ( sandev );
  err_alloc:
  err_no_uris:
-	if ( leak ) {
-		DBGC ( sandev, "EFIBLK %#02x nullified and leaked\n",
-		       sandev->drive );
-	}
+	if ( leak )
+		DBGC ( drive, "EFIBLK %#02x nullified and leaked\n", drive );
 	return rc;
 }
 
@@ -365,7 +368,7 @@ static void efi_block_unhook ( unsigned int drive ) {
 	/* Find SAN device */
 	sandev = sandev_find ( drive );
 	if ( ! sandev ) {
-		DBG ( "EFIBLK cannot find drive %#02x\n", drive );
+		DBGC ( drive, "EFIBLK %#02x is not a SAN drive\n", drive );
 		return;
 	}
 	block = sandev->priv;
@@ -377,8 +380,8 @@ static void efi_block_unhook ( unsigned int drive ) {
 			&efi_block_io_protocol_guid, &block->block_io,
 			&efi_device_path_protocol_guid, block->path,
 			NULL ) ) != 0 ) ) {
-		DBGC ( sandev, "EFIBLK %#02x could not uninstall protocols: "
-		       "%s\n", sandev->drive, strerror ( -EEFI ( efirc ) ) );
+		DBGC ( drive, "EFIBLK %#02x could not uninstall protocols: "
+		       "%s\n", drive, strerror ( -EEFI ( efirc ) ) );
 		leak = 1;
 	}
 	efi_nullify_block ( &block->block_io );
@@ -397,10 +400,8 @@ static void efi_block_unhook ( unsigned int drive ) {
 		sandev_put ( sandev );
 
 	/* Report leakage, if applicable */
-	if ( leak && ( ! efi_shutdown_in_progress ) ) {
-		DBGC ( sandev, "EFIBLK %#02x nullified and leaked\n",
-		       sandev->drive );
-	}
+	if ( leak && ( ! efi_shutdown_in_progress ) )
+		DBGC ( drive, "EFIBLK %#02x nullified and leaked\n", drive );
 }
 
 /** An installed ACPI table */
@@ -509,69 +510,364 @@ static int efi_block_describe ( void ) {
 }
 
 /**
- * Try booting from child device of EFI block device
+ * Open root directory within a filesystem
  *
- * @v sandev		SAN device
- * @v handle		EFI handle
+ * @v drive		Drive number
+ * @v handle		Filesystem handle
+ * @v root		Root directory file to fill in
+ * @ret rc		Return status code
+ */
+static int efi_block_root ( unsigned int drive, EFI_HANDLE handle,
+			    EFI_FILE_PROTOCOL **root ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	EFI_GUID *protocol = &efi_simple_file_system_protocol_guid;
+	union {
+		EFI_SIMPLE_FILE_SYSTEM_PROTOCOL *fs;
+		void *interface;
+	} u;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Open filesystem protocol */
+	if ( ( efirc = bs->OpenProtocol ( handle, protocol, &u.interface,
+					  efi_image_handle, handle,
+					  EFI_OPEN_PROTOCOL_GET_PROTOCOL ))!=0){
+		rc = -EEFI ( efirc );
+		DBGC ( drive, "EFIBLK %#02x could not open %s filesystem: %s\n",
+		       drive, efi_handle_name ( handle ), strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Open root volume */
+	if ( ( efirc = u.fs->OpenVolume ( u.fs, root ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( drive, "EFIBLK %#02x could not open %s root: %s\n",
+		       drive, efi_handle_name ( handle ), strerror ( rc ) );
+		goto err_volume;
+	}
+
+	/* Success */
+	rc = 0;
+
+ err_volume:
+	bs->CloseProtocol ( handle, protocol, efi_image_handle, handle );
+ err_open:
+	return rc;
+}
+
+/**
+ * Check for existence of a file within a filesystem
+ *
+ * @v drive		Drive number
+ * @v handle		Filesystem handle
+ * @v root		Root directory
  * @v filename		Filename (or NULL to use default)
- * @v image		Image handle to fill in
  * @ret rc		Return status code
  */
-static int efi_block_boot_image ( struct san_device *sandev, EFI_HANDLE handle,
-				  const char *filename, EFI_HANDLE *image ) {
+static int efi_block_filename ( unsigned int drive, EFI_HANDLE handle,
+				EFI_FILE_PROTOCOL *root,
+				const char *filename ) {
+	CHAR16 tmp[ filename ? ( strlen ( filename ) + 1 /* wNUL */ ) : 0 ];
+	CHAR16 *wname;
+	EFI_FILE_PROTOCOL *file;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Construct filename */
+	if ( filename ) {
+		efi_snprintf ( tmp, sizeof ( tmp ), "%s", filename );
+		wname = tmp;
+	} else {
+		wname = efi_block_boot_filename;
+	}
+
+	/* Try opening file */
+	if ( ( efirc = root->Open ( root, &file, wname,
+				    EFI_FILE_MODE_READ, 0 ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( drive, "EFIBLK %#02x could not open %s/%ls: %s\n",
+		       drive, efi_handle_name ( handle ), wname,
+		       strerror ( rc ) );
+		goto err_file;
+	}
+
+	/* Success */
+	rc = 0;
+
+	file->Close ( file );
+ err_file:
+	return rc;
+}
+
+/**
+ * Check for EFI block device filesystem label
+ *
+ * @v drive		Drive number
+ * @v root		Root directory
+ * @v label		Volume label
+ * @ret rc		Return status code
+ */
+static int efi_block_label ( unsigned int drive, EFI_FILE_PROTOCOL *root,
+			     const char *label ) {
+	EFI_FILE_SYSTEM_INFO *info;
+	UINTN size;
+	char *actual;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Get length of file system information */
+	size = 0;
+	root->GetInfo ( root, &efi_file_system_info_id, &size, NULL );
+
+	/* Allocate file system information */
+	info = malloc ( size );
+	if ( ! info ) {
+		rc = -ENOMEM;
+		goto err_alloc_info;
+	}
+
+	/* Get file system information */
+	if ( ( efirc = root->GetInfo ( root, &efi_file_system_info_id, &size,
+				       info ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( drive, "EFIBLK %#02x could not get filesystem info: "
+		       "%s\n", drive, strerror ( rc ) );
+		goto err_get_info;
+	}
+
+	/* Construct volume label for comparison */
+	if ( asprintf ( &actual, "%ls", info->VolumeLabel ) < 0 ) {
+		rc = -ENOMEM;
+		goto err_alloc_label;
+	}
+
+	/* Compare volume label */
+	if ( strcasecmp ( label, actual ) != 0 ) {
+		DBGC ( drive, "EFIBLK %#02x has wrong label \"%s\"\n",
+		       drive, actual );
+		rc = -ENOENT;
+		goto err_compare;
+	}
+
+	/* Success */
+	rc = 0;
+
+ err_compare:
+	free ( actual );
+ err_alloc_label:
+ err_get_info:
+	free ( info );
+ err_alloc_info:
+	return rc;
+}
+
+/**
+ * Check EFI block device filesystem match
+ *
+ * @v drive		Drive number
+ * @v handle		Filesystem handle
+ * @v path		Block device path
+ * @v config		Boot configuration parameters
+ * @v fspath		Filesystem device path to fill in
+ * @ret rc		Return status code
+ */
+static int efi_block_match ( unsigned int drive, EFI_HANDLE handle,
+			     EFI_DEVICE_PATH_PROTOCOL *path,
+			     struct san_boot_config *config,
+			     EFI_DEVICE_PATH_PROTOCOL **fspath ) {
 	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
-	struct efi_block_data *block = sandev->priv;
+	EFI_GUID *protocol = &efi_device_path_protocol_guid;
 	union {
 		EFI_DEVICE_PATH_PROTOCOL *path;
 		void *interface;
-	} path;
-	EFI_DEVICE_PATH_PROTOCOL *boot_path;
-	FILEPATH_DEVICE_PATH *filepath;
-	EFI_DEVICE_PATH_PROTOCOL *end;
-	size_t prefix_len;
-	size_t filepath_len;
-	size_t boot_path_len;
+	} u;
+	EFI_FILE *root;
+	union uuid guid;
 	EFI_STATUS efirc;
 	int rc;
 
 	/* Identify device path */
-	if ( ( efirc = bs->OpenProtocol ( handle,
-					  &efi_device_path_protocol_guid,
-					  &path.interface, efi_image_handle,
-					  handle,
+	if ( ( efirc = bs->OpenProtocol ( handle, protocol, &u.interface,
+					  efi_image_handle, handle,
 					  EFI_OPEN_PROTOCOL_GET_PROTOCOL ))!=0){
-		DBGC ( sandev, "EFIBLK %#02x found filesystem with no device "
-		       "path??", sandev->drive );
 		rc = -EEFI ( efirc );
-		goto err_open_device_path;
+		DBGC ( drive, "EFIBLK %#02x could not open %s device path: "
+		       "%s\n", drive, efi_handle_name ( handle ),
+		       strerror ( rc ) );
+		goto err_open;
 	}
+	*fspath = u.path;
 
-	/* Check if this device is a child of our block device */
-	prefix_len = efi_path_len ( block->path );
-	if ( memcmp ( path.path, block->path, prefix_len ) != 0 ) {
+	/* Check if filesystem is a child of this block device */
+	if ( memcmp ( u.path, path, efi_path_len ( path ) ) != 0 ) {
 		/* Not a child device */
 		rc = -ENOTTY;
+		DBGC2 ( drive, "EFIBLK %#02x is not parent of %s\n",
+			drive, efi_handle_name ( handle ) );
 		goto err_not_child;
 	}
-	DBGC ( sandev, "EFIBLK %#02x found child device %s\n",
-	       sandev->drive, efi_devpath_text ( path.path ) );
+	DBGC ( drive, "EFIBLK %#02x contains filesystem %s\n",
+	       drive, efi_devpath_text ( u.path ) );
+
+	/* Check if filesystem matches GUID, if applicable */
+	if ( config->uuid ) {
+		if ( ( rc = efi_path_guid ( u.path, &guid ) ) != 0 ) {
+			DBGC ( drive, "EFIBLK %#02x could not determine GUID: "
+			       "%s\n", drive, strerror ( rc ) );
+			goto err_no_guid;
+		}
+		if ( memcmp ( config->uuid, &guid, sizeof ( guid ) ) != 0 ) {
+			DBGC ( drive, "EFIBLK %#02x has wrong GUID %s\n",
+			       drive, uuid_ntoa ( &guid ) );
+			rc = -ENOENT;
+			goto err_wrong_guid;
+		}
+	}
+
+	/* Open root directory */
+	if ( ( rc = efi_block_root ( drive, handle, &root ) ) != 0 )
+		goto err_root;
+
+	/* Check if filesystem contains boot filename */
+	if ( ( rc = efi_block_filename ( drive, handle, root,
+					 config->filename ) ) != 0 ) {
+		goto err_filename;
+	}
+
+	/* Check if filesystem contains additional filename, if applicable */
+	if ( config->extra &&
+	     ( ( rc = efi_block_filename ( drive, handle, root,
+					   config->extra ) ) != 0 ) ) {
+		goto err_extra;
+	}
+
+	/* Check volume label, if applicable */
+	if ( config->label &&
+	     ( ( rc = efi_block_label ( drive, root,
+					config->label ) ) != 0 ) ) {
+		goto err_label;
+	}
+
+	/* Success */
+	rc = 0;
+
+ err_label:
+ err_extra:
+ err_filename:
+	root->Close ( root );
+ err_root:
+ err_wrong_guid:
+ err_no_guid:
+ err_not_child:
+	bs->CloseProtocol ( handle, protocol, efi_image_handle, handle );
+ err_open:
+	return rc;
+}
+
+/**
+ * Scan EFI block device for a matching filesystem
+ *
+ * @v drive		Drive number
+ * @v handle		Block device handle
+ * @v config		Boot configuration parameters
+ * @v fspath		Filesystem device path to fill in
+ * @ret rc		Return status code
+ */
+static int efi_block_scan ( unsigned int drive, EFI_HANDLE handle,
+			    struct san_boot_config *config,
+			    EFI_DEVICE_PATH_PROTOCOL **fspath ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	EFI_GUID *protocol = &efi_device_path_protocol_guid;
+	union {
+		EFI_DEVICE_PATH_PROTOCOL *path;
+		void *interface;
+	} u;
+	EFI_HANDLE *handles;
+	UINTN count;
+	unsigned int i;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Connect up possible file system drivers */
+	efi_block_connect ( drive, handle );
+
+	/* Identify device path */
+	if ( ( efirc = bs->OpenProtocol ( handle, protocol, &u.interface,
+					  efi_image_handle, handle,
+					  EFI_OPEN_PROTOCOL_GET_PROTOCOL ))!=0){
+		rc = -EEFI ( efirc );
+		DBGC ( drive, "EFIBLK %#02x could not open device path: %s\n",
+		       drive, strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Locate all Simple File System protocol handles */
+	if ( ( efirc = bs->LocateHandleBuffer (
+			ByProtocol, &efi_simple_file_system_protocol_guid,
+			NULL, &count, &handles ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( drive, "EFIBLK %#02x cannot locate file systems: %s\n",
+		       drive, strerror ( rc ) );
+		goto err_locate;
+	}
+
+	/* Scan for a matching filesystem */
+	rc = -ENOENT;
+	for ( i = 0 ; i < count ; i++ ) {
+
+		/* Check for a matching filesystem */
+		if ( ( rc = efi_block_match ( drive, handles[i], u.path,
+					      config, fspath ) ) != 0 )
+			continue;
+
+		break;
+	}
+
+	bs->FreePool ( handles );
+ err_locate:
+	bs->CloseProtocol ( handle, protocol, efi_image_handle, handle );
+ err_open:
+	return rc;
+}
+
+/**
+ * Boot from EFI block device filesystem boot image
+ *
+ * @v drive		Drive number
+ * @v fspath		Filesystem device path
+ * @v filename		Filename (or NULL to use default)
+ * @ret rc		Return status code
+ */
+static int efi_block_exec ( unsigned int drive,
+			    EFI_DEVICE_PATH_PROTOCOL *fspath,
+			    const char *filename ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	EFI_DEVICE_PATH_PROTOCOL *path;
+	FILEPATH_DEVICE_PATH *filepath;
+	EFI_DEVICE_PATH_PROTOCOL *end;
+	EFI_HANDLE image;
+	size_t fspath_len;
+	size_t filepath_len;
+	size_t path_len;
+	EFI_STATUS efirc;
+	int rc;
 
 	/* Construct device path for boot image */
-	end = efi_path_end ( path.path );
-	prefix_len = ( ( ( void * ) end ) - ( ( void * ) path.path ) );
+	end = efi_path_end ( fspath );
+	fspath_len = ( ( ( void * ) end ) - ( ( void * ) fspath ) );
 	filepath_len = ( SIZE_OF_FILEPATH_DEVICE_PATH +
 			 ( filename ?
 			   ( ( strlen ( filename ) + 1 /* NUL */ ) *
 			     sizeof ( filepath->PathName[0] ) ) :
 			   sizeof ( efi_block_boot_filename ) ) );
-	boot_path_len = ( prefix_len + filepath_len + sizeof ( *end ) );
-	boot_path = zalloc ( boot_path_len );
-	if ( ! boot_path ) {
+	path_len = ( fspath_len + filepath_len + sizeof ( *end ) );
+	path = zalloc ( path_len );
+	if ( ! path ) {
 		rc = -ENOMEM;
-		goto err_alloc_path;
+		goto err_alloc;
 	}
-	memcpy ( boot_path, path.path, prefix_len );
-	filepath = ( ( ( void * ) boot_path ) + prefix_len );
+	memcpy ( path, fspath, fspath_len );
+	filepath = ( ( ( void * ) path ) + fspath_len );
 	filepath->Header.Type = MEDIA_DEVICE_PATH;
 	filepath->Header.SubType = MEDIA_FILEPATH_DP;
 	filepath->Header.Length[0] = ( filepath_len & 0xff );
@@ -584,29 +880,93 @@ static int efi_block_boot_image ( struct san_device *sandev, EFI_HANDLE handle,
 	}
 	end = ( ( ( void * ) filepath ) + filepath_len );
 	efi_path_terminate ( end );
-	DBGC ( sandev, "EFIBLK %#02x trying to load %s\n",
-	       sandev->drive, efi_devpath_text ( boot_path ) );
+	DBGC ( drive, "EFIBLK %#02x trying to load %s\n",
+	       drive, efi_devpath_text ( path ) );
 
-	/* Try loading boot image from this device */
-	*image = NULL;
-	if ( ( efirc = bs->LoadImage ( FALSE, efi_image_handle, boot_path,
-				       NULL, 0, image ) ) != 0 ) {
+	/* Load image */
+	image = NULL;
+	if ( ( efirc = bs->LoadImage ( FALSE, efi_image_handle, path, NULL, 0,
+				       &image ) ) != 0 ) {
 		rc = -EEFI ( efirc );
-		DBGC ( sandev, "EFIBLK %#02x could not load image: %s\n",
-		       sandev->drive, strerror ( rc ) );
-		if ( efirc == EFI_SECURITY_VIOLATION )
-			bs->UnloadImage ( *image );
-		goto err_load_image;
+		DBGC ( drive, "EFIBLK %#02x could not load: %s\n",
+		       drive, strerror ( rc ) );
+		if ( efirc == EFI_SECURITY_VIOLATION ) {
+			goto err_load_security_violation;
+		} else {
+			goto err_load;
+		}
+	}
+
+	/* Start image */
+	efirc = bs->StartImage ( image, NULL, NULL );
+	rc = ( efirc ? -EEFI ( efirc ) : 0 );
+	DBGC ( drive, "EFIBLK %#02x boot image returned: %s\n",
+	       drive, strerror ( rc ) );
+
+ err_load_security_violation:
+	bs->UnloadImage ( image );
+ err_load:
+	free ( path );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Check that EFI block device is eligible for a local virtual drive number
+ *
+ * @v handle		Block device handle
+ * @ret rc		Return status code
+ *
+ * We assign virtual drive numbers for local (non-SAN) EFI block
+ * devices that represent complete disks, to provide roughly
+ * equivalent functionality to BIOS drive numbers.
+ */
+static int efi_block_local ( EFI_HANDLE handle ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	EFI_GUID *protocol = &efi_block_io_protocol_guid;
+	struct san_device *sandev;
+	struct efi_block_data *block;
+	union {
+		EFI_BLOCK_IO_PROTOCOL *blockio;
+		void *interface;
+	} u;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Check if handle belongs to a SAN device */
+	for_each_sandev ( sandev ) {
+		block = sandev->priv;
+		if ( handle == block->handle ) {
+			rc = -ENOTTY;
+			goto err_sandev;
+		}
+	}
+
+	/* Open block I/O protocol */
+	if ( ( efirc = bs->OpenProtocol ( handle, protocol, &u.interface,
+					  efi_image_handle, handle,
+					  EFI_OPEN_PROTOCOL_GET_PROTOCOL ))!=0){
+		rc = -EEFI ( efirc );
+		DBGC ( handle, "EFIBLK %s could not open block I/O: %s\n",
+		       efi_handle_name ( handle ), strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Do not assign drive numbers for partitions */
+	if ( u.blockio->Media->LogicalPartition ) {
+		rc = -ENOTTY;
+		DBGC2 ( handle, "EFLBLK %s is a partition\n",
+			efi_handle_name ( handle ) );
+		goto err_partition;
 	}
 
 	/* Success */
 	rc = 0;
 
- err_load_image:
-	free ( boot_path );
- err_alloc_path:
- err_not_child:
- err_open_device_path:
+ err_partition:
+	bs->CloseProtocol ( handle, protocol, efi_image_handle, handle );
+ err_open:
+ err_sandev:
 	return rc;
 }
 
@@ -614,72 +974,121 @@ static int efi_block_boot_image ( struct san_device *sandev, EFI_HANDLE handle,
  * Boot from EFI block device
  *
  * @v drive		Drive number
- * @v filename		Filename (or NULL to use default)
+ * @v config		Boot configuration parameters
  * @ret rc		Return status code
  */
-static int efi_block_boot ( unsigned int drive, const char *filename ) {
+static int efi_block_boot ( unsigned int drive,
+			    struct san_boot_config *config ) {
 	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
-	struct san_device *sandev;
+	EFI_DEVICE_PATH_PROTOCOL *fspath = NULL;
 	EFI_HANDLE *handles;
-	EFI_HANDLE image = NULL;
+	EFI_HANDLE handle;
 	UINTN count;
-	unsigned int i;
+	struct san_device *sandev;
+	struct efi_block_data *block;
+	unsigned int vdrive;
+	unsigned int index;
 	EFI_STATUS efirc;
 	int rc;
 
-	/* Find SAN device */
-	sandev = sandev_find ( drive );
-	if ( ! sandev ) {
-		DBG ( "EFIBLK cannot find drive %#02x\n", drive );
-		DBG ( "EFIBLK attempting local boot for drive %#02x, "
-		      "filename=%s\n", drive, filename );
-		rc = efi_boot_local( drive, filename );
-		if (rc)
-			rc = -ENODEV;
-
-		goto err_sandev_find;
-	}
 
 	/* Release SNP devices */
 	efi_snp_release();
 
-	/* Connect all possible protocols */
-	efi_block_connect ( sandev );
-
-	/* Locate all handles supporting the Simple File System protocol */
-	if ( ( efirc = bs->LocateHandleBuffer (
-			ByProtocol, &efi_simple_file_system_protocol_guid,
-			NULL, &count, &handles ) ) != 0 ) {
+	/* Locate all block I/O protocol handles */
+	if ( ( efirc = bs->LocateHandleBuffer ( ByProtocol,
+						&efi_block_io_protocol_guid,
+						NULL, &count,
+						&handles ) ) != 0 ) {
 		rc = -EEFI ( efirc );
-		DBGC ( sandev, "EFIBLK %#02x cannot locate file systems: %s\n",
-		       sandev->drive, strerror ( rc ) );
-		goto err_locate_file_systems;
+		DBGC ( drive, "EFIBLK %#02x cannot locate block I/O: %s\n",
+		       drive, strerror ( rc ) );
+		goto err_locate_block_io;
 	}
 
-	/* Try booting from any available child device containing a
-	 * suitable boot image.  This is something of a wild stab in
-	 * the dark, but should end up conforming to user expectations
-	 * most of the time.
-	 */
+	/* Try booting from the first matching block device, if any */
 	rc = -ENOENT;
-	for ( i = 0 ; i < count ; i++ ) {
-		if ( ( rc = efi_block_boot_image ( sandev, handles[i], filename,
-						   &image ) ) != 0 )
+	for ( vdrive = 0, index = 0 ; ; vdrive++ ) {
+
+		/* Identify next drive number and block I/O handle */
+		if ( ( sandev = sandev_next ( vdrive ) ) &&
+		     ( ( sandev->drive == vdrive ) ||
+		       ( sandev->drive <= SAN_DEFAULT_DRIVE ) ||
+		       ( index >= count ) ) ) {
+
+			/* There is a SAN drive that either:
+			 *
+			 * a) has the current virtual drive number, or
+			 * b) has a drive number below SAN_DEFAULT_DRIVE, or
+			 * c) has a drive number higher than any local drive
+			 *
+			 * Use this SAN drive, since the explicit SAN
+			 * drive numbering takes precedence over the
+			 * implicit local drive numbering.
+			 */
+			block = sandev->priv;
+			handle = block->handle;
+
+			/* Use SAN drive's explicit drive number */
+			vdrive = sandev->drive;
+			DBGC ( vdrive, "EFIBLK %#02x is SAN drive %s\n",
+			       vdrive, efi_handle_name ( handle ) );
+
+		} else if ( index < count ) {
+
+			/* There is no SAN drive meeting any of the
+			 * above criteria.  Try the next block I/O
+			 * handle.
+			 */
+			handle = handles[index++];
+
+			/* Check if this handle is eligible to be
+			 * given a local virtual drive number.
+			 *
+			 * Do not record this as the overall error
+			 * status, since it is not an interesting
+			 * error.
+			 */
+			if ( efi_block_local ( handle ) != 0 ) {
+				/* Do not consume virtual drive number */
+				vdrive--;
+				continue;
+			}
+
+			/* Use the current virtual drive number, with
+			 * a minimum of SAN_DEFAULT_DRIVE to match
+			 * typical BIOS drive numbering.
+			 */
+			if ( vdrive < SAN_DEFAULT_DRIVE )
+				vdrive = SAN_DEFAULT_DRIVE;
+			DBGC ( vdrive, "EFIBLK %#02x is local drive %s\n",
+			       vdrive, efi_handle_name ( handle ) );
+
+		} else {
+
+			/* No more SAN or local drives */
+			break;
+		}
+
+		/* Skip non-matching drives */
+		if ( drive && ( drive != vdrive ) )
 			continue;
-		DBGC ( sandev, "EFIBLK %#02x found boot image\n",
-		       sandev->drive );
-		efirc = bs->StartImage ( image, NULL, NULL );
-		rc = ( efirc ? -EEFI ( efirc ) : 0 );
-		bs->UnloadImage ( image );
-		DBGC ( sandev, "EFIBLK %#02x boot image returned: %s\n",
-		       sandev->drive, strerror ( rc ) );
+		DBGC ( vdrive, "EFIBLK %#02x attempting to boot\n", vdrive );
+
+		/* Scan for a matching filesystem within this drive */
+		if ( ( rc = efi_block_scan ( vdrive, handle, config,
+					     &fspath ) ) != 0 ) {
+			continue;
+		}
+
+		/* Attempt to boot from the matched filesystem */
+		rc = efi_block_exec ( vdrive, fspath, config->filename );
 		break;
 	}
 
 	bs->FreePool ( handles );
- err_locate_file_systems:
+ err_locate_block_io:
 	efi_snp_claim();
- err_sandev_find:
 	return rc;
 }
 
diff --git a/src/interface/efi/efi_debug.c b/src/interface/efi/efi_debug.c
index 1372776f..52efebe5 100644
--- a/src/interface/efi/efi_debug.c
+++ b/src/interface/efi/efi_debug.c
@@ -479,7 +479,7 @@ efi_devpath_text ( EFI_DEVICE_PATH_PROTOCOL *path ) {
 	}
 
 	/* Convert path to a textual representation */
-	wtext = efidpt->ConvertDevicePathToText ( path, TRUE, FALSE );
+	wtext = efidpt->ConvertDevicePathToText ( path, FALSE, FALSE );
 	if ( ! wtext )
 		return NULL;
 
@@ -665,10 +665,10 @@ efi_pecoff_debug_name ( EFI_LOADED_IMAGE_PROTOCOL *loaded ) {
 	snprintf ( buf, sizeof ( buf ), "%s", name );
 
 	/* Strip file suffix, if present */
-	if ( ( tmp = strrchr ( name, '.' ) ) != NULL )
+	if ( ( tmp = strrchr ( buf, '.' ) ) != NULL )
 		*tmp = '\0';
 
-	return name;
+	return buf;
 }
 
 /**
diff --git a/src/interface/efi/efi_path.c b/src/interface/efi/efi_path.c
index a78f97fc..d1e22eea 100644
--- a/src/interface/efi/efi_path.c
+++ b/src/interface/efi/efi_path.c
@@ -22,9 +22,11 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <stdlib.h>
 #include <stdarg.h>
 #include <string.h>
+#include <errno.h>
 #include <byteswap.h>
 #include <ipxe/netdevice.h>
 #include <ipxe/vlan.h>
+#include <ipxe/uuid.h>
 #include <ipxe/tcpip.h>
 #include <ipxe/uri.h>
 #include <ipxe/iscsi.h>
@@ -133,6 +135,47 @@ unsigned int efi_path_vlan ( EFI_DEVICE_PATH_PROTOCOL *path ) {
 }
 
 /**
+ * Get partition GUID from device path
+ *
+ * @v path		Device path
+ * @v guid		Partition GUID to fill in
+ * @ret rc		Return status code
+ */
+int efi_path_guid ( EFI_DEVICE_PATH_PROTOCOL *path, union uuid *guid ) {
+	EFI_DEVICE_PATH_PROTOCOL *next;
+	HARDDRIVE_DEVICE_PATH *hd;
+	int rc;
+
+	/* Search for most specific partition device path */
+	rc = -ENOENT;
+	for ( ; ( next = efi_path_next ( path ) ) ; path = next ) {
+
+		/* Skip non-harddrive device paths */
+		if ( path->Type != MEDIA_DEVICE_PATH )
+			continue;
+		if ( path->SubType != MEDIA_HARDDRIVE_DP )
+			continue;
+
+		/* Skip non-GUID signatures */
+		hd = container_of ( path, HARDDRIVE_DEVICE_PATH, Header );
+		if ( hd->SignatureType != SIGNATURE_TYPE_GUID )
+			continue;
+
+		/* Extract GUID */
+		memcpy ( guid, hd->Signature, sizeof ( *guid ) );
+		uuid_mangle ( guid );
+
+		/* Record success, but continue searching in case
+		 * there exists a more specific GUID (e.g. a partition
+		 * GUID rather than a disk GUID).
+		 */
+		rc = 0;
+	}
+
+	return rc;
+}
+
+/**
  * Concatenate EFI device paths
  *
  * @v ...		List of device paths (NULL terminated)
diff --git a/src/interface/efi/efi_shim.c b/src/interface/efi/efi_shim.c
index a46d79d0..d5419512 100644
--- a/src/interface/efi/efi_shim.c
+++ b/src/interface/efi/efi_shim.c
@@ -112,9 +112,6 @@ struct image_tag efi_shim __image_tag = {
 /** Original GetMemoryMap() function */
 static EFI_GET_MEMORY_MAP efi_shim_orig_get_memory_map;
 
-/** Original ExitBootServices() function */
-static EFI_EXIT_BOOT_SERVICES efi_shim_orig_exit_boot_services;
-
 /** Original SetVariable() function */
 static EFI_SET_VARIABLE efi_shim_orig_set_variable;
 
@@ -161,49 +158,6 @@ static void efi_shim_unlock ( void ) {
 }
 
 /**
- * Wrap GetMemoryMap()
- *
- * @v len		Memory map size
- * @v map		Memory map
- * @v key		Memory map key
- * @v desclen		Descriptor size
- * @v descver		Descriptor version
- * @ret efirc		EFI status code
- */
-static EFIAPI EFI_STATUS efi_shim_get_memory_map ( UINTN *len,
-						   EFI_MEMORY_DESCRIPTOR *map,
-						   UINTN *key, UINTN *desclen,
-						   UINT32 *descver ) {
-
-	/* Unlock shim */
-	if ( ! efi_shim_require_loader )
-		efi_shim_unlock();
-
-	/* Hand off to original GetMemoryMap() */
-	return efi_shim_orig_get_memory_map ( len, map, key, desclen,
-					      descver );
-}
-
-/**
- * Wrap ExitBootServices()
- *
- * @v handle		Image handle
- * @v key		Memory map key
- * @ret efirc		EFI status code
- */
-static EFIAPI EFI_STATUS efi_shim_exit_boot_services ( EFI_HANDLE handle,
-						       UINTN key ) {
-	EFI_RUNTIME_SERVICES *rs = efi_systab->RuntimeServices;
-
-	/* Restore original runtime services functions */
-	rs->SetVariable = efi_shim_orig_set_variable;
-	rs->GetVariable = efi_shim_orig_get_variable;
-
-	/* Hand off to original ExitBootServices() */
-	return efi_shim_orig_exit_boot_services ( handle, key );
-}
-
-/**
  * Wrap SetVariable()
  *
  * @v name		Variable name
@@ -271,6 +225,47 @@ efi_shim_get_variable ( CHAR16 *name, EFI_GUID *guid, UINT32 *attrs,
 }
 
 /**
+ * Wrap GetMemoryMap()
+ *
+ * @v len		Memory map size
+ * @v map		Memory map
+ * @v key		Memory map key
+ * @v desclen		Descriptor size
+ * @v descver		Descriptor version
+ * @ret efirc		EFI status code
+ */
+static EFIAPI EFI_STATUS efi_shim_get_memory_map ( UINTN *len,
+						   EFI_MEMORY_DESCRIPTOR *map,
+						   UINTN *key, UINTN *desclen,
+						   UINT32 *descver ) {
+	EFI_RUNTIME_SERVICES *rs = efi_systab->RuntimeServices;
+
+	/* Unlock shim */
+	if ( ! efi_shim_require_loader )
+		efi_shim_unlock();
+
+	/* Uninstall runtime services wrappers, if still installed */
+	if ( rs->SetVariable == efi_shim_set_variable ) {
+		rs->SetVariable = efi_shim_orig_set_variable;
+		DBGC ( &efi_shim, "SHIM uninstalled SetVariable() wrapper\n" );
+	} else if ( rs->SetVariable != efi_shim_orig_set_variable ) {
+		DBGC ( &efi_shim, "SHIM could not uninstall SetVariable() "
+		       "wrapper!\n" );
+	}
+	if ( rs->GetVariable == efi_shim_get_variable ) {
+		rs->GetVariable = efi_shim_orig_get_variable;
+		DBGC ( &efi_shim, "SHIM uninstalled GetVariable() wrapper\n" );
+	} else if ( rs->GetVariable != efi_shim_orig_get_variable ) {
+		DBGC ( &efi_shim, "SHIM could not uninstall GetVariable() "
+		       "wrapper!\n" );
+	}
+
+	/* Hand off to original GetMemoryMap() */
+	return efi_shim_orig_get_memory_map ( len, map, key, desclen,
+					      descver );
+}
+
+/**
  * Inhibit use of PXE base code
  *
  * @v handle		EFI handle
@@ -373,15 +368,14 @@ int efi_shim_install ( struct image *shim, EFI_HANDLE handle,
 
 	/* Record original boot and runtime services functions */
 	efi_shim_orig_get_memory_map = bs->GetMemoryMap;
-	efi_shim_orig_exit_boot_services = bs->ExitBootServices;
 	efi_shim_orig_set_variable = rs->SetVariable;
 	efi_shim_orig_get_variable = rs->GetVariable;
 
 	/* Wrap relevant boot and runtime services functions */
 	bs->GetMemoryMap = efi_shim_get_memory_map;
-	bs->ExitBootServices = efi_shim_exit_boot_services;
 	rs->SetVariable = efi_shim_set_variable;
 	rs->GetVariable = efi_shim_get_variable;
+	DBGC ( &efi_shim, "SHIM installed wrappers\n" );
 
 	return 0;
 }
@@ -396,7 +390,7 @@ void efi_shim_uninstall ( void ) {
 
 	/* Restore original boot and runtime services functions */
 	bs->GetMemoryMap = efi_shim_orig_get_memory_map;
-	bs->ExitBootServices = efi_shim_orig_exit_boot_services;
 	rs->SetVariable = efi_shim_orig_set_variable;
 	rs->GetVariable = efi_shim_orig_get_variable;
+	DBGC ( &efi_shim, "SHIM uninstalled wrappers\n" );
 }
diff --git a/src/interface/smbios/smbios.c b/src/interface/smbios/smbios.c
index 12a080da..fdd14499 100644
--- a/src/interface/smbios/smbios.c
+++ b/src/interface/smbios/smbios.c
@@ -42,7 +42,27 @@ static struct smbios smbios = {
 };
 
 /**
- * Scan for SMBIOS entry point structure
+ * Calculate SMBIOS entry point structure checksum
+ *
+ * @v start		Start address of region
+ * @v offset		Offset of SMBIOS entry point structure
+ * @v len		Length of entry point structure
+ * @ret sum		Byte checksum
+ */
+static uint8_t smbios_checksum ( userptr_t start, size_t offset, size_t len ) {
+	size_t end = ( offset + len );
+	uint8_t sum;
+	uint8_t byte;
+
+	for ( sum = 0 ; offset < end ; offset++ ) {
+		copy_from_user ( &byte, start, offset, sizeof ( byte ) );
+		sum += byte;
+	}
+	return sum;
+}
+
+/**
+ * Scan for SMBIOS 32-bit entry point structure
  *
  * @v start		Start address of region to scan
  * @v len		Length of region to scan
@@ -51,28 +71,20 @@ static struct smbios smbios = {
  */
 int find_smbios_entry ( userptr_t start, size_t len,
 			struct smbios_entry *entry ) {
-	uint8_t buf[256]; /* 256 is maximum length possible */
 	static size_t offset = 0; /* Avoid repeated attempts to locate SMBIOS */
-	size_t entry_len;
-	unsigned int i;
 	uint8_t sum;
 
 	/* Try to find SMBIOS */
-	for ( ; offset < len ; offset += 0x10 ) {
+	for ( ; ( offset + sizeof ( *entry ) ) <= len ; offset += 0x10 ) {
 
 		/* Read start of header and verify signature */
 		copy_from_user ( entry, start, offset, sizeof ( *entry ) );
 		if ( entry->signature != SMBIOS_SIGNATURE )
 			continue;
 
-		/* Read whole header and verify checksum */
-		entry_len = entry->len;
-		assert ( entry_len <= sizeof ( buf ) );
-		copy_from_user ( buf, start, offset, entry_len );
-		for ( i = 0, sum = 0 ; i < entry_len ; i++ ) {
-			sum += buf[i];
-		}
-		if ( sum != 0 ) {
+		/* Verify checksum */
+		if ( ( sum = smbios_checksum ( start, offset,
+					       entry->len ) ) != 0 ) {
 			DBG ( "SMBIOS at %08lx has bad checksum %02x\n",
 			      user_to_phys ( start, offset ), sum );
 			continue;
@@ -90,6 +102,46 @@ int find_smbios_entry ( userptr_t start, size_t len,
 }
 
 /**
+ * Scan for SMBIOS 64-bit entry point structure
+ *
+ * @v start		Start address of region to scan
+ * @v len		Length of region to scan
+ * @v entry		SMBIOS entry point structure to fill in
+ * @ret rc		Return status code
+ */
+int find_smbios3_entry ( userptr_t start, size_t len,
+			 struct smbios3_entry *entry ) {
+	static size_t offset = 0; /* Avoid repeated attempts to locate SMBIOS */
+	uint8_t sum;
+
+	/* Try to find SMBIOS */
+	for ( ; ( offset + sizeof ( *entry ) ) <= len ; offset += 0x10 ) {
+
+		/* Read start of header and verify signature */
+		copy_from_user ( entry, start, offset, sizeof ( *entry ) );
+		if ( entry->signature != SMBIOS3_SIGNATURE )
+			continue;
+
+		/* Verify checksum */
+		if ( ( sum = smbios_checksum ( start, offset,
+					       entry->len ) ) != 0 ) {
+			DBG ( "SMBIOS3 at %08lx has bad checksum %02x\n",
+			      user_to_phys ( start, offset ), sum );
+			continue;
+		}
+
+		/* Fill result structure */
+		DBG ( "Found SMBIOS3 v%d.%d entry point at %08lx\n",
+		      entry->major, entry->minor,
+		      user_to_phys ( start, offset ) );
+		return 0;
+	}
+
+	DBG ( "No SMBIOS3 found\n" );
+	return -ENODEV;
+}
+
+/**
  * Find SMBIOS strings terminator
  *
  * @v offset		Offset to start of strings
diff --git a/src/net/aoe.c b/src/net/aoe.c
index e785e897..dba4f51b 100644
--- a/src/net/aoe.c
+++ b/src/net/aoe.c
@@ -374,7 +374,7 @@ static void aoecmd_ata_cmd ( struct aoe_command *aoecmd,
 	struct aoeata *aoeata = &aoehdr->payload[0].ata;
 
 	/* Sanity check */
-	linker_assert ( AOE_FL_DEV_HEAD	== ATA_DEV_SLAVE, __fix_ata_h__ );
+	static_assert ( AOE_FL_DEV_HEAD == ATA_DEV_SLAVE );
 	assert ( len == ( sizeof ( *aoehdr ) + sizeof ( *aoeata ) +
 			  command->data_out_len ) );
 
diff --git a/src/net/eap.c b/src/net/eap.c
index beaeb61d..87327d72 100644
--- a/src/net/eap.c
+++ b/src/net/eap.c
@@ -23,7 +23,10 @@
 
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
+#include <stdlib.h>
 #include <errno.h>
+#include <string.h>
+#include <byteswap.h>
 #include <ipxe/netdevice.h>
 #include <ipxe/eap.h>
 
@@ -34,54 +37,183 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  */
 
 /**
+ * Transmit EAP response
+ *
+ * @v supplicant	EAP supplicant
+ * @v rsp		Response type data
+ * @v rsp_len		Length of response type data
+ * @ret rc		Return status code
+ */
+int eap_tx_response ( struct eap_supplicant *supplicant,
+		      const void *rsp, size_t rsp_len ) {
+	struct net_device *netdev = supplicant->netdev;
+	struct eap_message *msg;
+	size_t len;
+	int rc;
+
+	/* Allocate and populate response */
+	len = ( sizeof ( *msg ) + rsp_len );
+	msg = malloc ( len );
+	if ( ! msg ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	msg->hdr.code = EAP_CODE_RESPONSE;
+	msg->hdr.id = supplicant->id;
+	msg->hdr.len = htons ( len );
+	msg->type = supplicant->type;
+	memcpy ( msg->data, rsp, rsp_len );
+	DBGC ( netdev, "EAP %s Response id %#02x type %d\n",
+	       netdev->name, msg->hdr.id, msg->type );
+
+	/* Transmit response */
+	if ( ( rc = supplicant->tx ( supplicant, msg, len ) ) != 0 ) {
+		DBGC ( netdev, "EAP %s could not transmit: %s\n",
+		       netdev->name, strerror ( rc ) );
+		goto err_tx;
+	}
+
+ err_tx:
+	free ( msg );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Transmit EAP NAK
+ *
+ * @v supplicant	EAP supplicant
+ * @ret rc		Return status code
+ */
+static int eap_tx_nak ( struct eap_supplicant *supplicant ) {
+	struct net_device *netdev = supplicant->netdev;
+	unsigned int max = table_num_entries ( EAP_METHODS );
+	uint8_t methods[ max + 1 /* potential EAP_TYPE_NONE */ ];
+	unsigned int count = 0;
+	struct eap_method *method;
+
+	/* Populate methods list */
+	DBGC ( netdev, "EAP %s Nak offering types {", netdev->name );
+	for_each_table_entry ( method, EAP_METHODS ) {
+		if ( method->type > EAP_TYPE_NAK ) {
+			DBGC ( netdev, "%s%d",
+			       ( count ? ", " : "" ), method->type );
+			methods[count++] = method->type;
+		}
+	}
+	if ( ! count )
+		methods[count++] = EAP_TYPE_NONE;
+	DBGC ( netdev, "}\n" );
+	assert ( count <= max );
+
+	/* Transmit response */
+	supplicant->type = EAP_TYPE_NAK;
+	return eap_tx_response ( supplicant, methods, count );
+}
+
+/**
  * Handle EAP Request-Identity
  *
  * @v supplicant	EAP supplicant
+ * @v req		Request type data
+ * @v req_len		Length of request type data
  * @ret rc		Return status code
  */
-static int eap_rx_request_identity ( struct eap_supplicant *supplicant ) {
+static int eap_rx_identity ( struct eap_supplicant *supplicant,
+			     const void *req, size_t req_len ) {
 	struct net_device *netdev = supplicant->netdev;
+	void *rsp;
+	int rsp_len;
+	int rc;
 
 	/* Treat Request-Identity as blocking the link */
 	DBGC ( netdev, "EAP %s Request-Identity blocking link\n",
 	       netdev->name );
+	DBGC_HDA ( netdev, 0, req, req_len );
 	netdev_link_block ( netdev, EAP_BLOCK_TIMEOUT );
 
-	return 0;
+	/* Mark EAP as in progress */
+	supplicant->flags |= EAP_FL_ONGOING;
+
+	/* Construct response, if applicable */
+	rsp_len = fetch_raw_setting_copy ( netdev_settings ( netdev ),
+					   &username_setting, &rsp );
+	if ( rsp_len < 0 ) {
+		/* We have no identity to offer, so wait until the
+		 * switch times out and switches to MAC Authentication
+		 * Bypass (MAB).
+		 */
+		DBGC2 ( netdev, "EAP %s has no identity\n", netdev->name );
+		supplicant->flags |= EAP_FL_PASSIVE;
+		rc = 0;
+		goto no_response;
+	}
+
+	/* Transmit response */
+	if ( ( rc = eap_tx_response ( supplicant, rsp, rsp_len ) ) != 0 )
+		goto err_tx;
+
+ err_tx:
+	free ( rsp );
+ no_response:
+	return rc;
 }
 
+/** EAP Request-Identity method */
+struct eap_method eap_identity_method __eap_method = {
+	.type = EAP_TYPE_IDENTITY,
+	.rx = eap_rx_identity,
+};
+
 /**
  * Handle EAP Request
  *
  * @v supplicant	EAP supplicant
- * @v req		EAP request
+ * @v msg		EAP request
  * @v len		Length of EAP request
  * @ret rc		Return status code
  */
 static int eap_rx_request ( struct eap_supplicant *supplicant,
-			    const struct eap_request *req, size_t len ) {
+			    const struct eap_message *msg, size_t len ) {
 	struct net_device *netdev = supplicant->netdev;
+	struct eap_method *method;
+	const void *req;
+	size_t req_len;
 
-	/* Sanity check */
-	if ( len < sizeof ( *req ) ) {
+	/* Sanity checks */
+	if ( len < sizeof ( *msg ) ) {
 		DBGC ( netdev, "EAP %s underlength request:\n", netdev->name );
-		DBGC_HDA ( netdev, 0, req, len );
+		DBGC_HDA ( netdev, 0, msg, len );
+		return -EINVAL;
+	}
+	if ( len < ntohs ( msg->hdr.len ) ) {
+		DBGC ( netdev, "EAP %s truncated request:\n", netdev->name );
+		DBGC_HDA ( netdev, 0, msg, len );
 		return -EINVAL;
 	}
+	req = msg->data;
+	req_len = ( ntohs ( msg->hdr.len ) - sizeof ( *msg ) );
 
-	/* Mark authentication as incomplete */
-	supplicant->done = 0;
+	/* Record request details */
+	supplicant->id = msg->hdr.id;
+	supplicant->type = msg->type;
+	DBGC ( netdev, "EAP %s Request id %#02x type %d\n",
+	       netdev->name, msg->hdr.id, msg->type );
 
 	/* Handle according to type */
-	switch ( req->type ) {
-	case EAP_TYPE_IDENTITY:
-		return eap_rx_request_identity ( supplicant );
-	default:
-		DBGC ( netdev, "EAP %s requested type %d unknown:\n",
-		       netdev->name, req->type );
-		DBGC_HDA ( netdev, 0, req, len );
-		return -ENOTSUP;
+	for_each_table_entry ( method, EAP_METHODS ) {
+		if ( msg->type == method->type )
+			return method->rx ( supplicant, req, req_len );
 	}
+	DBGC ( netdev, "EAP %s requested type %d unknown:\n",
+	       netdev->name, msg->type );
+	DBGC_HDA ( netdev, 0, msg, len );
+
+	/* Send NAK if applicable */
+	if ( msg->type > EAP_TYPE_NAK )
+		return eap_tx_nak ( supplicant );
+
+	return -ENOTSUP;
 }
 
 /**
@@ -94,7 +226,7 @@ static int eap_rx_success ( struct eap_supplicant *supplicant ) {
 	struct net_device *netdev = supplicant->netdev;
 
 	/* Mark authentication as complete */
-	supplicant->done = 1;
+	supplicant->flags = EAP_FL_PASSIVE;
 
 	/* Mark link as unblocked */
 	DBGC ( netdev, "EAP %s Success\n", netdev->name );
@@ -113,7 +245,7 @@ static int eap_rx_failure ( struct eap_supplicant *supplicant ) {
 	struct net_device *netdev = supplicant->netdev;
 
 	/* Mark authentication as complete */
-	supplicant->done = 1;
+	supplicant->flags = EAP_FL_PASSIVE;
 
 	/* Record error */
 	DBGC ( netdev, "EAP %s Failure\n", netdev->name );
@@ -143,7 +275,10 @@ int eap_rx ( struct eap_supplicant *supplicant, const void *data,
 	/* Handle according to code */
 	switch ( eap->hdr.code ) {
 	case EAP_CODE_REQUEST:
-		return eap_rx_request ( supplicant, &eap->req, len );
+		return eap_rx_request ( supplicant, &eap->msg, len );
+	case EAP_CODE_RESPONSE:
+		DBGC2 ( netdev, "EAP %s ignoring response\n", netdev->name );
+		return 0;
 	case EAP_CODE_SUCCESS:
 		return eap_rx_success ( supplicant );
 	case EAP_CODE_FAILURE:
@@ -155,3 +290,9 @@ int eap_rx ( struct eap_supplicant *supplicant, const void *data,
 		return -ENOTSUP;
 	}
 }
+
+/* Drag in objects via eap_rx() */
+REQUIRING_SYMBOL ( eap_rx );
+
+/* Drag in EAP configuration */
+REQUIRE_OBJECT ( config_eap );
diff --git a/src/net/eap_md5.c b/src/net/eap_md5.c
new file mode 100644
index 00000000..0664174f
--- /dev/null
+++ b/src/net/eap_md5.c
@@ -0,0 +1,116 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <ipxe/md5.h>
+#include <ipxe/chap.h>
+#include <ipxe/eap.h>
+
+/** @file
+ *
+ * EAP MD5-Challenge authentication method
+ *
+ */
+
+/**
+ * Handle EAP MD5-Challenge
+ *
+ * @v supplicant	EAP supplicant
+ * @v req		Request type data
+ * @v req_len		Length of request type data
+ * @ret rc		Return status code
+ */
+static int eap_rx_md5 ( struct eap_supplicant *supplicant,
+			const void *req, size_t req_len ) {
+	struct net_device *netdev = supplicant->netdev;
+	const struct eap_md5 *md5req = req;
+	struct {
+		uint8_t len;
+		uint8_t value[MD5_DIGEST_SIZE];
+	} __attribute__ (( packed )) md5rsp;
+	struct chap_response chap;
+	void *secret;
+	int secret_len;
+	int rc;
+
+	/* Sanity checks */
+	if ( req_len < sizeof ( *md5req ) ) {
+		DBGC ( netdev, "EAP %s underlength MD5-Challenge:\n",
+		       netdev->name );
+		DBGC_HDA ( netdev, 0, req, req_len );
+		rc = -EINVAL;
+		goto err_sanity;
+	}
+	if ( ( req_len - sizeof ( *md5req ) ) < md5req->len ) {
+		DBGC ( netdev, "EAP %s truncated MD5-Challenge:\n",
+		       netdev->name );
+		DBGC_HDA ( netdev, 0, req, req_len );
+		rc = -EINVAL;
+		goto err_sanity;
+	}
+
+	/* Construct response */
+	if ( ( rc = chap_init ( &chap, &md5_algorithm ) ) != 0 ) {
+		DBGC ( netdev, "EAP %s could not initialise CHAP: %s\n",
+		       netdev->name, strerror ( rc ) );
+		goto err_chap;
+	}
+	chap_set_identifier ( &chap, supplicant->id );
+	secret_len = fetch_raw_setting_copy ( netdev_settings ( netdev ),
+					      &password_setting, &secret );
+	if ( secret_len < 0 ) {
+		rc = secret_len;
+		DBGC ( netdev, "EAP %s has no secret: %s\n",
+		       netdev->name, strerror ( rc ) );
+		goto err_secret;
+	}
+	chap_update ( &chap, secret, secret_len );
+	chap_update ( &chap, md5req->value, md5req->len );
+	chap_respond ( &chap );
+	assert ( chap.response_len == sizeof ( md5rsp.value ) );
+	md5rsp.len = sizeof ( md5rsp.value );
+	memcpy ( md5rsp.value, chap.response, sizeof ( md5rsp.value ) );
+
+	/* Transmit response */
+	if ( ( rc = eap_tx_response ( supplicant, &md5rsp,
+				      sizeof ( md5rsp ) ) ) != 0 )
+		goto err_tx;
+
+ err_tx:
+	free ( secret );
+ err_secret:
+	chap_finish ( &chap );
+ err_chap:
+ err_sanity:
+	return rc;
+}
+
+/** EAP MD5-Challenge method */
+struct eap_method eap_md5_method __eap_method = {
+	.type = EAP_TYPE_MD5,
+	.rx = eap_rx_md5,
+};
diff --git a/src/net/eap_mschapv2.c b/src/net/eap_mschapv2.c
new file mode 100644
index 00000000..0be62ed5
--- /dev/null
+++ b/src/net/eap_mschapv2.c
@@ -0,0 +1,251 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <byteswap.h>
+#include <ipxe/mschapv2.h>
+#include <ipxe/eap.h>
+
+/** @file
+ *
+ * EAP MS-CHAPv2 authentication method
+ *
+ * EAP-MSCHAPv2 was described in a draft RFC first published in 2002
+ * (draft-kamath-pppext-eap-mschapv2-02.txt).  The draft eventually
+ * expired in 2007 without becoming an official RFC, quite possibly
+ * because the protocol design was too ugly to be called an IETF
+ * standard.  It is, however, fairly widely used.
+ */
+
+/** An EAP MS-CHAPv2 request message */
+struct eap_mschapv2_request {
+	/** EAP-MSCHAPv2 header */
+	struct eap_mschapv2 hdr;
+	/** MS-CHAPv2 challenge length (fixed value) */
+	uint8_t len;
+	/** MS-CHAPv2 challenge */
+	struct mschapv2_challenge msg;
+} __attribute__ (( packed ));
+
+/** An EAP MS-CHAPv2 response message */
+struct eap_mschapv2_response {
+	/** EAP-MSCHAPv2 header */
+	struct eap_mschapv2 hdr;
+	/** MS-CHAPv2 response length (fixed value) */
+	uint8_t len;
+	/** MS-CHAPv2 response */
+	struct mschapv2_response msg;
+	/** User name */
+	char name[0];
+} __attribute__ (( packed ));
+
+/** An EAP MS-CHAPv2 success request message */
+struct eap_mschapv2_success_request {
+	/** EAP-MSCHAPv2 header */
+	struct eap_mschapv2 hdr;
+	/** Message */
+	char message[0];
+} __attribute__ (( packed ));
+
+/** An EAP MS-CHAPv2 success response message */
+struct eap_mschapv2_success_response {
+	/** Opcode */
+	uint8_t code;
+} __attribute__ (( packed ));
+
+/**
+ * Handle EAP MS-CHAPv2 request
+ *
+ * @v supplicant	EAP supplicant
+ * @v hdr		EAP-MSCHAPv2 header
+ * @v len		Message length
+ * @ret rc		Return status code
+ */
+static int eap_rx_mschapv2_request ( struct eap_supplicant *supplicant,
+				     const struct eap_mschapv2 *hdr,
+				     size_t len ) {
+	struct net_device *netdev = supplicant->netdev;
+	struct settings *settings = netdev_settings ( netdev );
+	const struct eap_mschapv2_request *msreq =
+		container_of ( hdr, struct eap_mschapv2_request, hdr );
+	struct eap_mschapv2_response *msrsp;
+	struct mschapv2_challenge peer;
+	char *username;
+	char *password;
+	int username_len;
+	int password_len;
+	size_t msrsp_len;
+	unsigned int i;
+	int rc;
+
+	/* Sanity check */
+	if ( len < sizeof ( *msreq ) ) {
+		DBGC ( netdev, "EAP %s underlength MS-CHAPv2 request\n",
+		       netdev->name );
+		DBGC_HDA ( netdev, 0, hdr, len );
+		rc = -EINVAL;
+		goto err_sanity;
+	}
+
+	/* Fetch username and password */
+	username_len = fetch_string_setting_copy ( settings, &username_setting,
+						   &username );
+	if ( username_len < 0 ) {
+		rc = username_len;
+		DBGC ( netdev, "EAP %s has no username: %s\n",
+		       netdev->name, strerror ( rc ) );
+		goto err_username;
+	}
+	password_len = fetch_string_setting_copy ( settings, &password_setting,
+						   &password );
+	if ( password_len < 0 ) {
+		rc = password_len;
+		DBGC ( netdev, "EAP %s has no password: %s\n",
+		       netdev->name, strerror ( rc ) );
+		goto err_password;
+	}
+
+	/* Construct a peer challenge.  We do not perform mutual
+	 * authentication, so this does not need to be strong.
+	 */
+	for ( i = 0 ; i < ( sizeof ( peer.byte ) /
+			    sizeof ( peer.byte[0] ) ) ; i++ ) {
+		peer.byte[i] = random();
+	}
+
+	/* Allocate response */
+	msrsp_len = ( sizeof ( *msrsp ) + username_len );
+	msrsp = malloc ( msrsp_len );
+	if ( ! msrsp ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Construct response */
+	msrsp->hdr.code = EAP_CODE_RESPONSE;
+	msrsp->hdr.id = msreq->hdr.id;
+	msrsp->hdr.len = htons ( msrsp_len );
+	msrsp->len = sizeof ( msrsp->msg );
+	mschapv2_response ( username, password, &msreq->msg, &peer,
+			    &msrsp->msg );
+	memcpy ( msrsp->name, username, username_len );
+
+	/* Send response */
+	if ( ( rc = eap_tx_response ( supplicant, msrsp, msrsp_len ) ) != 0 )
+		goto err_tx;
+
+ err_tx:
+	free ( msrsp );
+ err_alloc:
+	free ( password );
+ err_password:
+	free ( username );
+ err_username:
+ err_sanity:
+	return rc;
+}
+
+/**
+ * Handle EAP MS-CHAPv2 success request
+ *
+ * @v supplicant	EAP supplicant
+ * @v hdr		EAP-MSCHAPv2 header
+ * @v len		Message length
+ * @ret rc		Return status code
+ */
+static int eap_rx_mschapv2_success ( struct eap_supplicant *supplicant,
+				     const struct eap_mschapv2 *hdr,
+				     size_t len ) {
+	const struct eap_mschapv2_success_request *msreq =
+		container_of ( hdr, struct eap_mschapv2_success_request, hdr );
+	static const struct eap_mschapv2_success_response msrsp = {
+		.code = EAP_CODE_SUCCESS,
+	};
+
+	/* Sanity check */
+	assert ( len >= sizeof ( *msreq ) );
+
+	/* The success request contains the MS-CHAPv2 authenticator
+	 * response, which could potentially be used to verify that
+	 * the EAP authenticator also knew the password (or, at least,
+	 * the MD4 hash of the password).
+	 *
+	 * Our model for EAP does not encompass mutual authentication:
+	 * we will starting sending plaintext packets (e.g. DHCP
+	 * requests) over the link even before EAP completes, and our
+	 * only use for an EAP success is to mark the link as
+	 * unblocked.
+	 *
+	 * We therefore ignore the content of the success request and
+	 * just send back a success response, so that the EAP
+	 * authenticator will complete the process and send through
+	 * the real EAP success packet (which will, in turn, cause us
+	 * to unblock the link).
+	 */
+	return eap_tx_response ( supplicant, &msrsp, sizeof ( msrsp ) );
+}
+
+/**
+ * Handle EAP MS-CHAPv2
+ *
+ * @v supplicant	EAP supplicant
+ * @v req		Request type data
+ * @v req_len		Length of request type data
+ * @ret rc		Return status code
+ */
+static int eap_rx_mschapv2 ( struct eap_supplicant *supplicant,
+			     const void *req, size_t req_len ) {
+	struct net_device *netdev = supplicant->netdev;
+	const struct eap_mschapv2 *hdr = req;
+
+	/* Sanity check */
+	if ( req_len < sizeof ( *hdr ) ) {
+		DBGC ( netdev, "EAP %s underlength MS-CHAPv2:\n",
+		       netdev->name );
+		DBGC_HDA ( netdev, 0, req, req_len );
+		return -EINVAL;
+	}
+
+	/* Handle according to opcode */
+	switch ( hdr->code ) {
+	case EAP_CODE_REQUEST:
+		return eap_rx_mschapv2_request ( supplicant, hdr, req_len );
+	case EAP_CODE_SUCCESS:
+		return eap_rx_mschapv2_success ( supplicant, hdr, req_len );
+	default:
+		DBGC ( netdev, "EAP %s unsupported MS-CHAPv2 opcode %d\n",
+		       netdev->name, hdr->code );
+		DBGC_HDA ( netdev, 0, req, req_len );
+		return -ENOTSUP;
+	}
+}
+
+/** EAP MS-CHAPv2 method */
+struct eap_method eap_mschapv2_method __eap_method = {
+	.type = EAP_TYPE_MSCHAPV2,
+	.rx = eap_rx_mschapv2,
+};
diff --git a/src/net/eapol.c b/src/net/eapol.c
index 1b843e89..8b09ca23 100644
--- a/src/net/eapol.c
+++ b/src/net/eapol.c
@@ -49,37 +49,6 @@ static const uint8_t eapol_mac[ETH_ALEN] = {
 };
 
 /**
- * Update EAPoL supplicant state
- *
- * @v supplicant	EAPoL supplicant
- * @v timeout		Timer ticks until next EAPoL-Start (if applicable)
- */
-static void eapol_update ( struct eapol_supplicant *supplicant,
-			   unsigned long timeout ) {
-	struct net_device *netdev = supplicant->eap.netdev;
-
-	/* Check device and EAP state */
-	if ( netdev_is_open ( netdev ) && netdev_link_ok ( netdev ) ) {
-		if ( supplicant->eap.done ) {
-
-			/* EAP has completed: stop sending EAPoL-Start */
-			stop_timer ( &supplicant->timer );
-
-		} else if ( ! timer_running ( &supplicant->timer ) ) {
-
-			/* EAP has not yet begun: start sending EAPoL-Start */
-			start_timer_fixed ( &supplicant->timer, timeout );
-		}
-
-	} else {
-
-		/* Not ready: clear completion and stop sending EAPoL-Start */
-		supplicant->eap.done = 0;
-		stop_timer ( &supplicant->timer );
-	}
-}
-
-/**
  * Process EAPoL packet
  *
  * @v iobuf		I/O buffer
@@ -186,8 +155,20 @@ static int eapol_eap_rx ( struct eapol_supplicant *supplicant,
 		goto drop;
 	}
 
-	/* Update supplicant state */
-	eapol_update ( supplicant, EAPOL_START_INTERVAL );
+	/* Update EAPoL-Start transmission timer */
+	if ( supplicant->eap.flags & EAP_FL_PASSIVE ) {
+		/* Stop sending EAPoL-Start */
+		if ( timer_running ( &supplicant->timer ) ) {
+			DBGC ( netdev, "EAPOL %s becoming passive\n",
+			       netdev->name );
+		}
+		stop_timer ( &supplicant->timer );
+	} else if ( supplicant->eap.flags & EAP_FL_ONGOING ) {
+		/* Delay EAPoL-Start until after next expected packet */
+		DBGC ( netdev, "EAPOL %s deferring Start\n", netdev->name );
+		start_timer_fixed ( &supplicant->timer, EAP_WAIT_TIMEOUT );
+		supplicant->count = 0;
+	}
 
  drop:
 	free_iob ( iobuf );
@@ -270,6 +251,12 @@ static void eapol_expired ( struct retry_timer *timer, int fail __unused ) {
 		container_of ( timer, struct eapol_supplicant, timer );
 	struct net_device *netdev = supplicant->eap.netdev;
 
+	/* Stop transmitting after maximum number of attempts */
+	if ( supplicant->count++ >= EAPOL_START_COUNT ) {
+		DBGC ( netdev, "EAPOL %s giving up\n", netdev->name );
+		return;
+	}
+
 	/* Schedule next transmission */
 	start_timer_fixed ( timer, EAPOL_START_INTERVAL );
 
@@ -309,15 +296,36 @@ static int eapol_probe ( struct net_device *netdev, void *priv ) {
  * @v netdev		Network device
  * @v priv		Private data
  */
-static void eapol_notify ( struct net_device *netdev __unused, void *priv ) {
+static void eapol_notify ( struct net_device *netdev, void *priv ) {
 	struct eapol_supplicant *supplicant = priv;
 
 	/* Ignore non-EAPoL devices */
 	if ( ! supplicant->eap.netdev )
 		return;
 
-	/* Update supplicant state */
-	eapol_update ( supplicant, 0 );
+	/* Terminate and reset EAP when link goes down */
+	if ( ! ( netdev_is_open ( netdev ) && netdev_link_ok ( netdev ) ) ) {
+		if ( timer_running ( &supplicant->timer ) ) {
+			DBGC ( netdev, "EAPOL %s shutting down\n",
+			       netdev->name );
+		}
+		supplicant->eap.flags = 0;
+		stop_timer ( &supplicant->timer );
+		return;
+	}
+
+	/* Do nothing if EAP is already in progress */
+	if ( timer_running ( &supplicant->timer ) )
+		return;
+
+	/* Do nothing if EAP has already finished transmitting */
+	if ( supplicant->eap.flags & EAP_FL_PASSIVE )
+		return;
+
+	/* Otherwise, start sending EAPoL-Start */
+	start_timer_nodelay ( &supplicant->timer );
+	supplicant->count = 0;
+	DBGC ( netdev, "EAPOL %s starting up\n", netdev->name );
 }
 
 /** EAPoL driver */
diff --git a/src/net/tls.c b/src/net/tls.c
index 000a8a78..5f89be45 100644
--- a/src/net/tls.c
+++ b/src/net/tls.c
@@ -158,6 +158,10 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define EINFO_ENOTSUP_VERSION						\
 	__einfo_uniqify ( EINFO_ENOTSUP, 0x04,				\
 			  "Unsupported protocol version" )
+#define ENOTSUP_CURVE __einfo_error ( EINFO_ENOTSUP_CURVE )
+#define EINFO_ENOTSUP_CURVE						\
+	__einfo_uniqify ( EINFO_ENOTSUP, 0x05,				\
+			  "Unsupported elliptic curve" )
 #define EPERM_ALERT __einfo_error ( EINFO_EPERM_ALERT )
 #define EINFO_EPERM_ALERT						\
 	__einfo_uniqify ( EINFO_EPERM, 0x01,				\
@@ -1044,6 +1048,35 @@ tls_signature_hash_digest ( struct tls_signature_hash_id code ) {
 
 /******************************************************************************
  *
+ * Ephemeral Elliptic Curve Diffie-Hellman key exchange
+ *
+ ******************************************************************************
+ */
+
+/** Number of supported named curves */
+#define TLS_NUM_NAMED_CURVES table_num_entries ( TLS_NAMED_CURVES )
+
+/**
+ * Identify named curve
+ *
+ * @v named_curve	Named curve specification
+ * @ret curve		Named curve, or NULL
+ */
+static struct tls_named_curve *
+tls_find_named_curve ( unsigned int named_curve ) {
+	struct tls_named_curve *curve;
+
+	/* Identify named curve */
+	for_each_table_entry ( curve, TLS_NAMED_CURVES ) {
+		if ( curve->code == named_curve )
+			return curve;
+	}
+
+	return NULL;
+}
+
+/******************************************************************************
+ *
  * Record handling
  *
  ******************************************************************************
@@ -1122,6 +1155,67 @@ static int tls_client_hello ( struct tls_connection *tls,
 	struct tls_session *session = tls->session;
 	size_t name_len = strlen ( session->name );
 	struct {
+		uint16_t type;
+		uint16_t len;
+		struct {
+			uint16_t len;
+			struct {
+				uint8_t type;
+				uint16_t len;
+				uint8_t name[name_len];
+			} __attribute__ (( packed )) list[1];
+		} __attribute__ (( packed )) data;
+	} __attribute__ (( packed )) *server_name_ext;
+	struct {
+		uint16_t type;
+		uint16_t len;
+		struct {
+			uint8_t max;
+		} __attribute__ (( packed )) data;
+	} __attribute__ (( packed )) *max_fragment_length_ext;
+	struct {
+		uint16_t type;
+		uint16_t len;
+		struct {
+			uint16_t len;
+			struct tls_signature_hash_id
+				code[TLS_NUM_SIG_HASH_ALGORITHMS];
+		} __attribute__ (( packed )) data;
+	} __attribute__ (( packed )) *signature_algorithms_ext;
+	struct {
+		uint16_t type;
+		uint16_t len;
+		struct {
+			uint8_t len;
+			uint8_t data[ tls->secure_renegotiation ?
+				      sizeof ( tls->verify.client ) :0 ];
+		} __attribute__ (( packed )) data;
+	} __attribute__ (( packed )) *renegotiation_info_ext;
+	struct {
+		uint16_t type;
+		uint16_t len;
+		struct {
+			uint8_t data[session->ticket_len];
+		} __attribute__ (( packed )) data;
+	} __attribute__ (( packed )) *session_ticket_ext;
+	struct {
+		uint16_t type;
+		uint16_t len;
+		struct {
+			uint16_t len;
+			uint16_t code[TLS_NUM_NAMED_CURVES];
+		} __attribute__ (( packed )) data;
+	} __attribute__ (( packed )) *named_curve_ext;
+	struct {
+		typeof ( *server_name_ext ) server_name;
+		typeof ( *max_fragment_length_ext ) max_fragment_length;
+		typeof ( *signature_algorithms_ext ) signature_algorithms;
+		typeof ( *renegotiation_info_ext ) renegotiation_info;
+		typeof ( *session_ticket_ext ) session_ticket;
+		typeof ( *named_curve_ext )
+			named_curve[TLS_NUM_NAMED_CURVES ? 1 : 0];
+	} __attribute__ (( packed )) *extensions;
+	struct {
 		uint32_t type_length;
 		uint16_t version;
 		uint8_t random[32];
@@ -1132,45 +1226,11 @@ static int tls_client_hello ( struct tls_connection *tls,
 		uint8_t compression_methods_len;
 		uint8_t compression_methods[1];
 		uint16_t extensions_len;
-		struct {
-			uint16_t server_name_type;
-			uint16_t server_name_len;
-			struct {
-				uint16_t len;
-				struct {
-					uint8_t type;
-					uint16_t len;
-					uint8_t name[name_len];
-				} __attribute__ (( packed )) list[1];
-			} __attribute__ (( packed )) server_name;
-			uint16_t max_fragment_length_type;
-			uint16_t max_fragment_length_len;
-			struct {
-				uint8_t max;
-			} __attribute__ (( packed )) max_fragment_length;
-			uint16_t signature_algorithms_type;
-			uint16_t signature_algorithms_len;
-			struct {
-				uint16_t len;
-				struct tls_signature_hash_id
-					code[TLS_NUM_SIG_HASH_ALGORITHMS];
-			} __attribute__ (( packed )) signature_algorithms;
-			uint16_t renegotiation_info_type;
-			uint16_t renegotiation_info_len;
-			struct {
-				uint8_t len;
-				uint8_t data[ tls->secure_renegotiation ?
-					      sizeof ( tls->verify.client ) :0];
-			} __attribute__ (( packed )) renegotiation_info;
-			uint16_t session_ticket_type;
-			uint16_t session_ticket_len;
-			struct {
-				uint8_t data[session->ticket_len];
-			} __attribute__ (( packed )) session_ticket;
-		} __attribute__ (( packed )) extensions;
+		typeof ( *extensions ) extensions;
 	} __attribute__ (( packed )) hello;
 	struct tls_cipher_suite *suite;
 	struct tls_signature_hash_algorithm *sighash;
+	struct tls_named_curve *curve;
 	unsigned int i;
 
 	/* Construct record */
@@ -1188,43 +1248,66 @@ static int tls_client_hello ( struct tls_connection *tls,
 		hello.cipher_suites[i++] = suite->code;
 	hello.compression_methods_len = sizeof ( hello.compression_methods );
 	hello.extensions_len = htons ( sizeof ( hello.extensions ) );
-	hello.extensions.server_name_type = htons ( TLS_SERVER_NAME );
-	hello.extensions.server_name_len
-		= htons ( sizeof ( hello.extensions.server_name ) );
-	hello.extensions.server_name.len
-		= htons ( sizeof ( hello.extensions.server_name.list ) );
-	hello.extensions.server_name.list[0].type = TLS_SERVER_NAME_HOST_NAME;
-	hello.extensions.server_name.list[0].len
-		= htons ( sizeof ( hello.extensions.server_name.list[0].name ));
-	memcpy ( hello.extensions.server_name.list[0].name, session->name,
-		 sizeof ( hello.extensions.server_name.list[0].name ) );
-	hello.extensions.max_fragment_length_type
-		= htons ( TLS_MAX_FRAGMENT_LENGTH );
-	hello.extensions.max_fragment_length_len
-		= htons ( sizeof ( hello.extensions.max_fragment_length ) );
-	hello.extensions.max_fragment_length.max
-		= TLS_MAX_FRAGMENT_LENGTH_4096;
-	hello.extensions.signature_algorithms_type
-		= htons ( TLS_SIGNATURE_ALGORITHMS );
-	hello.extensions.signature_algorithms_len
-		= htons ( sizeof ( hello.extensions.signature_algorithms ) );
-	hello.extensions.signature_algorithms.len
-		= htons ( sizeof ( hello.extensions.signature_algorithms.code));
+	extensions = &hello.extensions;
+
+	/* Construct server name extension */
+	server_name_ext = &extensions->server_name;
+	server_name_ext->type = htons ( TLS_SERVER_NAME );
+	server_name_ext->len = htons ( sizeof ( server_name_ext->data ) );
+	server_name_ext->data.len
+		= htons ( sizeof ( server_name_ext->data.list ) );
+	server_name_ext->data.list[0].type = TLS_SERVER_NAME_HOST_NAME;
+	server_name_ext->data.list[0].len
+		= htons ( sizeof ( server_name_ext->data.list[0].name ) );
+	memcpy ( server_name_ext->data.list[0].name, session->name,
+		 sizeof ( server_name_ext->data.list[0].name ) );
+
+	/* Construct maximum fragment length extension */
+	max_fragment_length_ext = &extensions->max_fragment_length;
+	max_fragment_length_ext->type = htons ( TLS_MAX_FRAGMENT_LENGTH );
+	max_fragment_length_ext->len
+		= htons ( sizeof ( max_fragment_length_ext->data ) );
+	max_fragment_length_ext->data.max = TLS_MAX_FRAGMENT_LENGTH_4096;
+
+	/* Construct supported signature algorithms extension */
+	signature_algorithms_ext = &extensions->signature_algorithms;
+	signature_algorithms_ext->type = htons ( TLS_SIGNATURE_ALGORITHMS );
+	signature_algorithms_ext->len
+		= htons ( sizeof ( signature_algorithms_ext->data ) );
+	signature_algorithms_ext->data.len
+		= htons ( sizeof ( signature_algorithms_ext->data.code ) );
 	i = 0 ; for_each_table_entry ( sighash, TLS_SIG_HASH_ALGORITHMS )
-		hello.extensions.signature_algorithms.code[i++] = sighash->code;
-	hello.extensions.renegotiation_info_type
-		= htons ( TLS_RENEGOTIATION_INFO );
-	hello.extensions.renegotiation_info_len
-		= htons ( sizeof ( hello.extensions.renegotiation_info ) );
-	hello.extensions.renegotiation_info.len
-		= sizeof ( hello.extensions.renegotiation_info.data );
-	memcpy ( hello.extensions.renegotiation_info.data, tls->verify.client,
-		 sizeof ( hello.extensions.renegotiation_info.data ) );
-	hello.extensions.session_ticket_type = htons ( TLS_SESSION_TICKET );
-	hello.extensions.session_ticket_len
-		= htons ( sizeof ( hello.extensions.session_ticket ) );
-	memcpy ( hello.extensions.session_ticket.data, session->ticket,
-		 sizeof ( hello.extensions.session_ticket.data ) );
+		signature_algorithms_ext->data.code[i++] = sighash->code;
+
+	/* Construct renegotiation information extension */
+	renegotiation_info_ext = &extensions->renegotiation_info;
+	renegotiation_info_ext->type = htons ( TLS_RENEGOTIATION_INFO );
+	renegotiation_info_ext->len
+		= htons ( sizeof ( renegotiation_info_ext->data ) );
+	renegotiation_info_ext->data.len
+		= sizeof ( renegotiation_info_ext->data.data );
+	memcpy ( renegotiation_info_ext->data.data, tls->verify.client,
+		 sizeof ( renegotiation_info_ext->data.data ) );
+
+	/* Construct session ticket extension */
+	session_ticket_ext = &extensions->session_ticket;
+	session_ticket_ext->type = htons ( TLS_SESSION_TICKET );
+	session_ticket_ext->len
+		= htons ( sizeof ( session_ticket_ext->data ) );
+	memcpy ( session_ticket_ext->data.data, session->ticket,
+		 sizeof ( session_ticket_ext->data.data ) );
+
+	/* Construct named curves extension, if applicable */
+	if ( sizeof ( extensions->named_curve ) ) {
+		named_curve_ext = &extensions->named_curve[0];
+		named_curve_ext->type = htons ( TLS_NAMED_CURVE );
+		named_curve_ext->len
+			= htons ( sizeof ( named_curve_ext->data ) );
+		named_curve_ext->data.len
+			= htons ( sizeof ( named_curve_ext->data.code ) );
+		i = 0 ; for_each_table_entry ( curve, TLS_NAMED_CURVES )
+			named_curve_ext->data.code[i++] = curve->code;
+	}
 
 	return action ( tls, &hello, sizeof ( hello ) );
 }
@@ -1336,13 +1419,6 @@ static int tls_send_client_key_exchange_pubkey ( struct tls_connection *tls ) {
 	tls_generate_master_secret ( tls, &pre_master_secret,
 				     sizeof ( pre_master_secret ) );
 
-	/* Generate keys */
-	if ( ( rc = tls_generate_keys ( tls ) ) != 0 ) {
-		DBGC ( tls, "TLS %p could not generate keys: %s\n",
-		       tls, strerror ( rc ) );
-		return rc;
-	}
-
 	/* Encrypt pre-master secret using server's public key */
 	memset ( &key_xchg, 0, sizeof ( key_xchg ) );
 	len = pubkey_encrypt ( pubkey, cipherspec->pubkey_ctx,
@@ -1374,21 +1450,18 @@ struct tls_key_exchange_algorithm tls_pubkey_exchange_algorithm = {
 };
 
 /**
- * Transmit Client Key Exchange record using DHE key exchange
+ * Verify Diffie-Hellman parameter signature
  *
  * @v tls		TLS connection
+ * @v param_len		Diffie-Hellman parameter length
  * @ret rc		Return status code
  */
-static int tls_send_client_key_exchange_dhe ( struct tls_connection *tls ) {
+static int tls_verify_dh_params ( struct tls_connection *tls,
+				  size_t param_len ) {
 	struct tls_cipherspec *cipherspec = &tls->tx_cipherspec_pending;
 	struct pubkey_algorithm *pubkey;
 	struct digest_algorithm *digest;
 	int use_sig_hash = tls_version ( tls, TLS_VERSION_TLS_1_2 );
-	uint8_t private[ sizeof ( tls->client_random.random ) ];
-	const struct {
-		uint16_t len;
-		uint8_t data[0];
-	} __attribute__ (( packed )) *dh_val[3];
 	const struct {
 		struct tls_signature_hash_id sig_hash[use_sig_hash];
 		uint16_t signature_len;
@@ -1396,29 +1469,14 @@ static int tls_send_client_key_exchange_dhe ( struct tls_connection *tls ) {
 	} __attribute__ (( packed )) *sig;
 	const void *data;
 	size_t remaining;
-	size_t frag_len;
-	unsigned int i;
 	int rc;
 
-	/* Parse ServerKeyExchange */
-	data = tls->server_key;
-	remaining = tls->server_key_len;
-	for ( i = 0 ; i < ( sizeof ( dh_val ) / sizeof ( dh_val[0] ) ) ; i++ ){
-		dh_val[i] = data;
-		if ( ( sizeof ( *dh_val[i] ) > remaining ) ||
-		     ( ntohs ( dh_val[i]->len ) > ( remaining -
-						    sizeof ( *dh_val[i] ) ) )){
-			DBGC ( tls, "TLS %p received underlength "
-			       "ServerKeyExchange\n", tls );
-			DBGC_HDA ( tls, 0, tls->server_key,
-				   tls->server_key_len );
-			rc = -EINVAL_KEY_EXCHANGE;
-			goto err_header;
-		}
-		frag_len = ( sizeof ( *dh_val[i] ) + ntohs ( dh_val[i]->len ));
-		data += frag_len;
-		remaining -= frag_len;
-	}
+	/* Signature follows parameters */
+	assert ( param_len <= tls->server_key_len );
+	data = ( tls->server_key + param_len );
+	remaining = ( tls->server_key_len - param_len );
+
+	/* Parse signature from ServerKeyExchange */
 	sig = data;
 	if ( ( sizeof ( *sig ) > remaining ) ||
 	     ( ntohs ( sig->signature_len ) > ( remaining -
@@ -1426,8 +1484,7 @@ static int tls_send_client_key_exchange_dhe ( struct tls_connection *tls ) {
 		DBGC ( tls, "TLS %p received underlength ServerKeyExchange\n",
 		       tls );
 		DBGC_HDA ( tls, 0, tls->server_key, tls->server_key_len );
-		rc = -EINVAL_KEY_EXCHANGE;
-		goto err_header;
+		return -EINVAL_KEY_EXCHANGE;
 	}
 
 	/* Identify signature and hash algorithm */
@@ -1437,15 +1494,13 @@ static int tls_send_client_key_exchange_dhe ( struct tls_connection *tls ) {
 		if ( ( ! pubkey ) || ( ! digest ) ) {
 			DBGC ( tls, "TLS %p ServerKeyExchange unsupported "
 			       "signature and hash algorithm\n", tls );
-			rc = -ENOTSUP_SIG_HASH;
-			goto err_sig_hash;
+			return -ENOTSUP_SIG_HASH;
 		}
 		if ( pubkey != cipherspec->suite->pubkey ) {
 			DBGC ( tls, "TLS %p ServerKeyExchange incorrect "
 			       "signature algorithm %s (expected %s)\n", tls,
 			       pubkey->name, cipherspec->suite->pubkey->name );
-			rc = -EPERM_KEY_EXCHANGE;
-			goto err_sig_hash;
+			return -EPERM_KEY_EXCHANGE;
 		}
 	} else {
 		pubkey = cipherspec->suite->pubkey;
@@ -1465,8 +1520,7 @@ static int tls_send_client_key_exchange_dhe ( struct tls_connection *tls ) {
 				sizeof ( tls->client_random ) );
 		digest_update ( digest, ctx, tls->server_random,
 				sizeof ( tls->server_random ) );
-		digest_update ( digest, ctx, tls->server_key,
-				( tls->server_key_len - remaining ) );
+		digest_update ( digest, ctx, tls->server_key, param_len );
 		digest_final ( digest, ctx, hash );
 
 		/* Verify signature */
@@ -1477,10 +1531,56 @@ static int tls_send_client_key_exchange_dhe ( struct tls_connection *tls ) {
 			       "verification\n", tls );
 			DBGC_HDA ( tls, 0, tls->server_key,
 				   tls->server_key_len );
-			rc = -EPERM_KEY_EXCHANGE;
-			goto err_verify;
+			return -EPERM_KEY_EXCHANGE;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * Transmit Client Key Exchange record using DHE key exchange
+ *
+ * @v tls		TLS connection
+ * @ret rc		Return status code
+ */
+static int tls_send_client_key_exchange_dhe ( struct tls_connection *tls ) {
+	uint8_t private[ sizeof ( tls->client_random.random ) ];
+	const struct {
+		uint16_t len;
+		uint8_t data[0];
+	} __attribute__ (( packed )) *dh_val[3];
+	const void *data;
+	size_t remaining;
+	size_t frag_len;
+	size_t param_len;
+	unsigned int i;
+	int rc;
+
+	/* Parse ServerKeyExchange */
+	data = tls->server_key;
+	remaining = tls->server_key_len;
+	for ( i = 0 ; i < ( sizeof ( dh_val ) / sizeof ( dh_val[0] ) ) ; i++ ){
+		dh_val[i] = data;
+		if ( ( sizeof ( *dh_val[i] ) > remaining ) ||
+		     ( ntohs ( dh_val[i]->len ) > ( remaining -
+						    sizeof ( *dh_val[i] ) ) )){
+			DBGC ( tls, "TLS %p received underlength "
+			       "ServerKeyExchange\n", tls );
+			DBGC_HDA ( tls, 0, tls->server_key,
+				   tls->server_key_len );
+			rc = -EINVAL_KEY_EXCHANGE;
+			goto err_header;
 		}
+		frag_len = ( sizeof ( *dh_val[i] ) + ntohs ( dh_val[i]->len ));
+		data += frag_len;
+		remaining -= frag_len;
 	}
+	param_len = ( tls->server_key_len - remaining );
+
+	/* Verify parameter signature */
+	if ( ( rc = tls_verify_dh_params ( tls, param_len ) ) != 0 )
+		goto err_verify;
 
 	/* Generate Diffie-Hellman private key */
 	if ( ( rc = tls_generate_random ( tls, private,
@@ -1540,13 +1640,6 @@ static int tls_send_client_key_exchange_dhe ( struct tls_connection *tls ) {
 		/* Generate master secret */
 		tls_generate_master_secret ( tls, pre_master_secret, len );
 
-		/* Generate keys */
-		if ( ( rc = tls_generate_keys ( tls ) ) != 0 ) {
-			DBGC ( tls, "TLS %p could not generate keys: %s\n",
-			       tls, strerror ( rc ) );
-			goto err_generate_keys;
-		}
-
 		/* Transmit Client Key Exchange record */
 		if ( ( rc = tls_send_handshake ( tls, key_xchg,
 						 sizeof ( *key_xchg ) ) ) !=0){
@@ -1554,14 +1647,12 @@ static int tls_send_client_key_exchange_dhe ( struct tls_connection *tls ) {
 		}
 
 	err_send_handshake:
-	err_generate_keys:
 	err_dhe_key:
 		free ( dynamic );
 	}
  err_alloc:
  err_random:
  err_verify:
- err_sig_hash:
  err_header:
 	return rc;
 }
@@ -1573,6 +1664,119 @@ struct tls_key_exchange_algorithm tls_dhe_exchange_algorithm = {
 };
 
 /**
+ * Transmit Client Key Exchange record using ECDHE key exchange
+ *
+ * @v tls		TLS connection
+ * @ret rc		Return status code
+ */
+static int tls_send_client_key_exchange_ecdhe ( struct tls_connection *tls ) {
+	struct tls_named_curve *curve;
+	const struct {
+		uint8_t curve_type;
+		uint16_t named_curve;
+		uint8_t public_len;
+		uint8_t public[0];
+	} __attribute__ (( packed )) *ecdh;
+	size_t param_len;
+	int rc;
+
+	/* Parse ServerKeyExchange record */
+	ecdh = tls->server_key;
+	if ( ( sizeof ( *ecdh ) > tls->server_key_len ) ||
+	     ( ecdh->public_len > ( tls->server_key_len - sizeof ( *ecdh ) ))){
+		DBGC ( tls, "TLS %p received underlength ServerKeyExchange\n",
+		       tls );
+		DBGC_HDA ( tls, 0, tls->server_key, tls->server_key_len );
+		return -EINVAL_KEY_EXCHANGE;
+	}
+	param_len = ( sizeof ( *ecdh ) + ecdh->public_len );
+
+	/* Verify parameter signature */
+	if ( ( rc = tls_verify_dh_params ( tls, param_len ) ) != 0 )
+		return rc;
+
+	/* Identify named curve */
+	if ( ecdh->curve_type != TLS_NAMED_CURVE_TYPE ) {
+		DBGC ( tls, "TLS %p unsupported curve type %d\n",
+		       tls, ecdh->curve_type );
+		DBGC_HDA ( tls, 0, tls->server_key, tls->server_key_len );
+		return -ENOTSUP_CURVE;
+	}
+	curve = tls_find_named_curve ( ecdh->named_curve );
+	if ( ! curve ) {
+		DBGC ( tls, "TLS %p unsupported named curve %d\n",
+		       tls, ntohs ( ecdh->named_curve ) );
+		DBGC_HDA ( tls, 0, tls->server_key, tls->server_key_len );
+		return -ENOTSUP_CURVE;
+	}
+
+	/* Check key length */
+	if ( ecdh->public_len != curve->curve->keysize ) {
+		DBGC ( tls, "TLS %p invalid %s key\n",
+		       tls, curve->curve->name );
+		DBGC_HDA ( tls, 0, tls->server_key, tls->server_key_len );
+		return -EINVAL_KEY_EXCHANGE;
+	}
+
+	/* Construct pre-master secret and ClientKeyExchange record */
+	{
+		size_t len = curve->curve->keysize;
+		uint8_t private[len];
+		uint8_t pre_master_secret[len];
+		struct {
+			uint32_t type_length;
+			uint8_t public_len;
+			uint8_t public[len];
+		} __attribute__ (( packed )) key_xchg;
+
+		/* Generate ephemeral private key */
+		if ( ( rc = tls_generate_random ( tls, private,
+						  sizeof ( private ) ) ) != 0){
+			return rc;
+		}
+
+		/* Calculate pre-master secret */
+		if ( ( rc = elliptic_multiply ( curve->curve,
+						ecdh->public, private,
+						pre_master_secret ) ) != 0 ) {
+			DBGC ( tls, "TLS %p could not exchange ECDHE key: %s\n",
+			       tls, strerror ( rc ) );
+			return rc;
+		}
+
+		/* Generate master secret */
+		tls_generate_master_secret ( tls, pre_master_secret, len );
+
+		/* Generate Client Key Exchange record */
+		key_xchg.type_length =
+			( cpu_to_le32 ( TLS_CLIENT_KEY_EXCHANGE ) |
+			  htonl ( sizeof ( key_xchg ) -
+				  sizeof ( key_xchg.type_length ) ) );
+		key_xchg.public_len = len;
+		if ( ( rc = elliptic_multiply ( curve->curve, NULL, private,
+						key_xchg.public ) ) != 0 ) {
+			DBGC ( tls, "TLS %p could not generate ECDHE key: %s\n",
+			       tls, strerror ( rc ) );
+			return rc;
+		}
+
+		/* Transmit Client Key Exchange record */
+		if ( ( rc = tls_send_handshake ( tls, &key_xchg,
+						 sizeof ( key_xchg ) ) ) !=0){
+			return rc;
+		}
+	}
+
+	return 0;
+}
+
+/** Ephemeral Elliptic Curve Diffie-Hellman key exchange algorithm */
+struct tls_key_exchange_algorithm tls_ecdhe_exchange_algorithm = {
+	.name = "ecdhe",
+	.exchange = tls_send_client_key_exchange_ecdhe,
+};
+
+/**
  * Transmit Client Key Exchange record
  *
  * @v tls		TLS connection
@@ -1581,9 +1785,23 @@ struct tls_key_exchange_algorithm tls_dhe_exchange_algorithm = {
 static int tls_send_client_key_exchange ( struct tls_connection *tls ) {
 	struct tls_cipherspec *cipherspec = &tls->tx_cipherspec_pending;
 	struct tls_cipher_suite *suite = cipherspec->suite;
+	int rc;
 
 	/* Transmit Client Key Exchange record via key exchange algorithm */
-	return suite->exchange->exchange ( tls );
+	if ( ( rc = suite->exchange->exchange ( tls ) ) != 0 ) {
+		DBGC ( tls, "TLS %p could not exchange keys: %s\n",
+		       tls, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Generate keys from master secret */
+	if ( ( rc = tls_generate_keys ( tls ) ) != 0 ) {
+		DBGC ( tls, "TLS %p could not generate keys: %s\n",
+		       tls, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
 }
 
 /**
@@ -2727,9 +2945,9 @@ static int tls_send_plaintext ( struct tls_connection *tls, unsigned int type,
 	} __attribute__ (( packed )) iv;
 	struct tls_auth_header authhdr;
 	struct tls_header *tlshdr;
-	void *plaintext = NULL;
-	size_t plaintext_len = len;
-	struct io_buffer *ciphertext = NULL;
+	void *plaintext;
+	size_t plaintext_len;
+	struct io_buffer *ciphertext;
 	size_t ciphertext_len;
 	size_t padding_len;
 	uint8_t mac[digest->digestsize];
@@ -2738,7 +2956,10 @@ static int tls_send_plaintext ( struct tls_connection *tls, unsigned int type,
 
 	/* Construct initialisation vector */
 	memcpy ( iv.fixed, cipherspec->fixed_iv, sizeof ( iv.fixed ) );
-	tls_generate_random ( tls, iv.record, sizeof ( iv.record ) );
+	if ( ( rc = tls_generate_random ( tls, iv.record,
+					  sizeof ( iv.record ) ) ) != 0 ) {
+		goto err_random;
+	}
 
 	/* Construct authentication data */
 	authhdr.seq = cpu_to_be64 ( tls->tx_seq );
@@ -2747,7 +2968,7 @@ static int tls_send_plaintext ( struct tls_connection *tls, unsigned int type,
 	authhdr.header.length = htons ( len );
 
 	/* Calculate padding length */
-	plaintext_len += suite->mac_len;
+	plaintext_len = ( len + suite->mac_len );
 	if ( is_block_cipher ( cipher ) ) {
 		padding_len = ( ( ( cipher->blocksize - 1 ) &
 				  -( plaintext_len + 1 ) ) + 1 );
@@ -2762,7 +2983,7 @@ static int tls_send_plaintext ( struct tls_connection *tls, unsigned int type,
 		DBGC ( tls, "TLS %p could not allocate %zd bytes for "
 		       "plaintext\n", tls, plaintext_len );
 		rc = -ENOMEM_TX_PLAINTEXT;
-		goto done;
+		goto err_plaintext;
 	}
 
 	/* Assemble plaintext */
@@ -2796,7 +3017,7 @@ static int tls_send_plaintext ( struct tls_connection *tls, unsigned int type,
 		DBGC ( tls, "TLS %p could not allocate %zd bytes for "
 		       "ciphertext\n", tls, ciphertext_len );
 		rc = -ENOMEM_TX_CIPHERTEXT;
-		goto done;
+		goto err_ciphertext;
 	}
 
 	/* Assemble ciphertext */
@@ -2821,15 +3042,22 @@ static int tls_send_plaintext ( struct tls_connection *tls, unsigned int type,
 				       iob_disown ( ciphertext ) ) ) != 0 ) {
 		DBGC ( tls, "TLS %p could not deliver ciphertext: %s\n",
 		       tls, strerror ( rc ) );
-		goto done;
+		goto err_deliver;
 	}
 
 	/* Update TX state machine to next record */
 	tls->tx_seq += 1;
 
- done:
-	free ( plaintext );
+	assert ( plaintext == NULL );
+	assert ( ciphertext == NULL );
+	return 0;
+
+ err_deliver:
 	free_iob ( ciphertext );
+ err_ciphertext:
+	free ( plaintext );
+ err_plaintext:
+ err_random:
 	return rc;
 }
 
diff --git a/src/net/validator.c b/src/net/validator.c
index 693d4464..69c0df33 100644
--- a/src/net/validator.c
+++ b/src/net/validator.c
@@ -57,8 +57,7 @@ struct validator_action {
 	/** Name */
 	const char *name;
 	/** Action to take upon completed transfer */
-	int ( * done ) ( struct validator *validator, const void *data,
-			 size_t len );
+	void ( * done ) ( struct validator *validator, int rc );
 };
 
 /** A certificate validator */
@@ -72,6 +71,40 @@ struct validator {
 
 	/** Process */
 	struct process process;
+	/** Most relevant status code
+	 *
+	 * The cross-signed certificate mechanism may attempt several
+	 * downloads as it works its way up the provided partial chain
+	 * to locate a suitable cross-signed certificate with which to
+	 * complete the chain.
+	 *
+	 * Some of these download or validation attempts may fail for
+	 * uninteresting reasons (i.e. because a cross-signed
+	 * certificate has never existed for that link in the chain).
+	 *
+	 * We must therefore keep track of the most relevant error
+	 * that has occurred, in order to be able to report a
+	 * meaningful overall status to the user.
+	 *
+	 * As a concrete example: consider the case of an expired OCSP
+	 * signer for an intermediate certificate.  This will cause
+	 * OCSP validation to fail for that intermediate certificate,
+	 * and this is the error that should eventually be reported to
+	 * the user.  We do not want to instead report the
+	 * uninteresting fact that no cross-signed certificate was
+	 * found for the remaining links in the chain, nor do we want
+	 * to report just a generic "OCSP required" error.
+	 *
+	 * We record the most relevant status code whenever a
+	 * definitely relevant error occurs, and clear it whenever we
+	 * successfully make forward progress (e.g. by completing
+	 * OCSP, or by adding new cross-signed certificates).
+	 *
+	 * When we subsequently attempt to validate the chain, we
+	 * report the most relevant error status code (if recorded),
+	 * otherwise we report the validation error itself.
+	 */
+	int rc;
 
 	/** Root of trust (or NULL to use default) */
 	struct x509_root *root;
@@ -84,13 +117,15 @@ struct validator {
 
 	/** Current action */
 	const struct validator_action *action;
-	/** Current certificate
+	/** Current certificate (for progress reporting)
 	 *
 	 * This will always be present within the certificate chain
 	 * and so this pointer does not hold a reference to the
 	 * certificate.
 	 */
 	struct x509_certificate *cert;
+	/** Current link within certificate chain */
+	struct x509_link *link;
 };
 
 /**
@@ -196,17 +231,36 @@ static const char crosscert_default[] = CROSSCERT;
  * Append cross-signing certificates to certificate chain
  *
  * @v validator		Certificate validator
- * @v data		Raw cross-signing certificate data
- * @v len		Length of raw data
+ * @v rc		Completion status code
  * @ret rc		Return status code
  */
-static int validator_append ( struct validator *validator,
-			      const void *data, size_t len ) {
+static void validator_append ( struct validator *validator, int rc ) {
 	struct asn1_cursor cursor;
 	struct x509_chain *certs;
 	struct x509_certificate *cert;
-	struct x509_certificate *last;
-	int rc;
+	struct x509_link *link;
+	struct x509_link *prev;
+
+	/* Check for errors */
+	if ( rc != 0 ) {
+		DBGC ( validator, "VALIDATOR %p \"%s\" could not download ",
+		       validator, validator_name ( validator ) );
+		DBGC ( validator, "\"%s\" cross-signature: %s\n",
+		       x509_name ( validator->cert ), strerror ( rc ) );
+		/* If the overall validation is going to fail, then we
+		 * will end up attempting multiple downloads for
+		 * non-existent cross-signed certificates as we work
+		 * our way up the certificate chain.  Do not record
+		 * these as relevant errors, since we want to
+		 * eventually report whichever much more relevant
+		 * error occurred previously.
+		 */
+		goto err_irrelevant;
+	}
+	DBGC ( validator, "VALIDATOR %p \"%s\" downloaded ",
+	       validator, validator_name ( validator ) );
+	DBGC ( validator, "\"%s\" cross-signature\n",
+	       x509_name ( validator->cert ) );
 
 	/* Allocate certificate list */
 	certs = x509_alloc_chain();
@@ -216,8 +270,8 @@ static int validator_append ( struct validator *validator,
 	}
 
 	/* Initialise cursor */
-	cursor.data = data;
-	cursor.len = len;
+	cursor.data = validator->buffer.data;
+	cursor.len = validator->buffer.len;
 
 	/* Enter certificateSet */
 	if ( ( rc = asn1_enter ( &cursor, ASN1_SET ) ) != 0 ) {
@@ -230,14 +284,14 @@ static int validator_append ( struct validator *validator,
 	/* Add each certificate to list */
 	while ( cursor.len ) {
 
-		/* Add certificate to chain */
+		/* Add certificate to list */
 		if ( ( rc = x509_append_raw ( certs, cursor.data,
 					      cursor.len ) ) != 0 ) {
 			DBGC ( validator, "VALIDATOR %p \"%s\" could not "
 			       "append certificate: %s\n", validator,
 			       validator_name ( validator ), strerror ( rc) );
 			DBGC_HDA ( validator, 0, cursor.data, cursor.len );
-			return rc;
+			goto err_append_raw;
 		}
 		cert = x509_last ( certs );
 		DBGC ( validator, "VALIDATOR %p \"%s\" found certificate ",
@@ -248,8 +302,12 @@ static int validator_append ( struct validator *validator,
 		asn1_skip_any ( &cursor );
 	}
 
+	/* Truncate existing certificate chain at current link */
+	link = validator->link;
+	assert ( link->flags & X509_LINK_FL_CROSSED );
+	x509_truncate ( validator->chain, link );
+
 	/* Append certificates to chain */
-	last = x509_last ( validator->chain );
 	if ( ( rc = x509_auto_append ( validator->chain, certs ) ) != 0 ) {
 		DBGC ( validator, "VALIDATOR %p \"%s\" could not append "
 		       "certificates: %s\n", validator,
@@ -257,26 +315,31 @@ static int validator_append ( struct validator *validator,
 		goto err_auto_append;
 	}
 
-	/* Check that at least one certificate has been added */
-	if ( last == x509_last ( validator->chain ) ) {
-		DBGC ( validator, "VALIDATOR %p \"%s\" failed to append any "
-		       "applicable certificates\n", validator,
-		       validator_name ( validator ) );
-		rc = -EACCES;
-		goto err_no_progress;
+	/* Record that a cross-signed certificate download has already
+	 * been performed for all but the last of the appended
+	 * certificates.  (It may be necessary to perform a further
+	 * download to complete the chain, if this download did not
+	 * extend all the way to a root of trust.)
+	 */
+	prev = NULL;
+	list_for_each_entry_continue ( link, &validator->chain->links, list ) {
+		if ( prev )
+			prev->flags |= X509_LINK_FL_CROSSED;
+		prev = link;
 	}
 
-	/* Drop reference to certificate list */
-	x509_chain_put ( certs );
-
-	return 0;
+	/* Success */
+	rc = 0;
 
- err_no_progress:
  err_auto_append:
+ err_append_raw:
  err_certificateset:
 	x509_chain_put ( certs );
  err_alloc_certs:
-	return rc;
+	validator->rc = rc;
+ err_irrelevant:
+	/* Do not record irrelevant errors */
+	return;
 }
 
 /** Cross-signing certificate download validator action */
@@ -289,11 +352,12 @@ static const struct validator_action validator_crosscert = {
  * Start download of cross-signing certificate
  *
  * @v validator		Certificate validator
- * @v cert		X.509 certificate
+ * @v link		Link in certificate chain
  * @ret rc		Return status code
  */
 static int validator_start_download ( struct validator *validator,
-				      struct x509_certificate *cert ) {
+				      struct x509_link *link ) {
+	struct x509_certificate *cert = link->cert;
 	const struct asn1_cursor *issuer = &cert->issuer.raw;
 	const char *crosscert;
 	char *crosscert_copy;
@@ -336,6 +400,7 @@ static int validator_start_download ( struct validator *validator,
 	/* Set completion handler */
 	validator->action = &validator_crosscert;
 	validator->cert = cert;
+	validator->link = link;
 
 	/* Open URI */
 	if ( ( rc = xfer_open_uri_string ( &validator->xfer,
@@ -346,14 +411,20 @@ static int validator_start_download ( struct validator *validator,
 		goto err_open_uri_string;
 	}
 
+	/* Free temporary allocations */
+	free ( uri_string );
+	free ( crosscert_copy );
+
 	/* Success */
-	rc = 0;
+	return 0;
 
+	intf_restart ( &validator->xfer, rc );
  err_open_uri_string:
 	free ( uri_string );
  err_alloc_uri_string:
  err_check_uri_string:
 	free ( crosscert_copy );
+	validator->rc = rc;
 	return rc;
 }
 
@@ -367,21 +438,27 @@ static int validator_start_download ( struct validator *validator,
  * Validate OCSP response
  *
  * @v validator		Certificate validator
- * @v data		Raw OCSP response
- * @v len		Length of raw data
- * @ret rc		Return status code
+ * @v rc		Completion status code
  */
-static int validator_ocsp_validate ( struct validator *validator,
-				     const void *data, size_t len ) {
+static void validator_ocsp_validate ( struct validator *validator, int rc ) {
+	const void *data = validator->buffer.data;
+	size_t len = validator->buffer.len;
 	time_t now;
-	int rc;
+
+	/* Check for errors */
+	if ( rc != 0 ) {
+		DBGC ( validator, "VALIDATOR %p \"%s\" could not fetch OCSP "
+		       "response: %s\n", validator,
+		       validator_name ( validator ), strerror ( rc ) );
+		goto err_status;
+	}
 
 	/* Record OCSP response */
 	if ( ( rc = ocsp_response ( validator->ocsp, data, len ) ) != 0 ) {
 		DBGC ( validator, "VALIDATOR %p \"%s\" could not record OCSP "
 		       "response: %s\n", validator,
-		       validator_name ( validator ),strerror ( rc ) );
-		return rc;
+		       validator_name ( validator ), strerror ( rc ) );
+		goto err_response;
 	}
 
 	/* Validate OCSP response */
@@ -390,14 +467,20 @@ static int validator_ocsp_validate ( struct validator *validator,
 		DBGC ( validator, "VALIDATOR %p \"%s\" could not validate "
 		       "OCSP response: %s\n", validator,
 		       validator_name ( validator ), strerror ( rc ) );
-		return rc;
+		goto err_validate;
 	}
 
-	/* Drop reference to OCSP check */
+	/* Success */
+	DBGC ( validator, "VALIDATOR %p \"%s\" checked ",
+	       validator, validator_name ( validator ) );
+	DBGC ( validator, "\"%s\" via OCSP\n", x509_name ( validator->cert ) );
+
+ err_validate:
+ err_response:
+ err_status:
 	ocsp_put ( validator->ocsp );
 	validator->ocsp = NULL;
-
-	return 0;
+	validator->rc = rc;
 }
 
 /** OCSP validator action */
@@ -426,7 +509,7 @@ static int validator_start_ocsp ( struct validator *validator,
 		DBGC ( validator, "VALIDATOR %p \"%s\" could not create OCSP "
 		       "check: %s\n", validator, validator_name ( validator ),
 		       strerror ( rc ) );
-		return rc;
+		goto err_check;
 	}
 
 	/* Set completion handler */
@@ -444,10 +527,18 @@ static int validator_start_ocsp ( struct validator *validator,
 		DBGC ( validator, "VALIDATOR %p \"%s\" could not open %s: "
 		       "%s\n", validator, validator_name ( validator ),
 		       uri_string, strerror ( rc ) );
-		return rc;
+		goto err_open;
 	}
 
 	return 0;
+
+	intf_restart ( &validator->xfer, rc );
+ err_open:
+	ocsp_put ( validator->ocsp );
+	validator->ocsp = NULL;
+ err_check:
+	validator->rc = rc;
+	return rc;
 }
 
 /****************************************************************************
@@ -466,34 +557,18 @@ static void validator_xfer_close ( struct validator *validator, int rc ) {
 
 	/* Close data transfer interface */
 	intf_restart ( &validator->xfer, rc );
-
-	/* Check for errors */
-	if ( rc != 0 ) {
-		DBGC ( validator, "VALIDATOR %p \"%s\" transfer failed: %s\n",
-		       validator, validator_name ( validator ),
-		       strerror ( rc ) );
-		goto err_transfer;
-	}
 	DBGC2 ( validator, "VALIDATOR %p \"%s\" transfer complete\n",
 		validator, validator_name ( validator ) );
 
 	/* Process completed download */
 	assert ( validator->action != NULL );
-	if ( ( rc = validator->action->done ( validator, validator->buffer.data,
-					      validator->buffer.len ) ) != 0 )
-		goto err_append;
+	validator->action->done ( validator, rc );
 
 	/* Free downloaded data */
 	xferbuf_free ( &validator->buffer );
 
 	/* Resume validation process */
 	process_add ( &validator->process );
-
-	return;
-
- err_append:
- err_transfer:
-	validator_finished ( validator, rc );
 }
 
 /**
@@ -515,7 +590,7 @@ static int validator_xfer_deliver ( struct validator *validator,
 		DBGC ( validator, "VALIDATOR %p \"%s\" could not receive "
 		       "data: %s\n", validator, validator_name ( validator ),
 		       strerror ( rc ) );
-		validator_finished ( validator, rc );
+		validator_xfer_close ( validator, rc );
 		return rc;
 	}
 
@@ -544,10 +619,10 @@ static struct interface_descriptor validator_xfer_desc =
  * @v validator		Certificate validator
  */
 static void validator_step ( struct validator *validator ) {
+	struct x509_chain *chain = validator->chain;
 	struct x509_link *link;
+	struct x509_link *prev;
 	struct x509_certificate *cert;
-	struct x509_certificate *issuer = NULL;
-	struct x509_certificate *last;
 	time_t now;
 	int rc;
 
@@ -556,57 +631,109 @@ static void validator_step ( struct validator *validator ) {
 	 * previously.
 	 */
 	now = time ( NULL );
-	if ( ( rc = x509_validate_chain ( validator->chain, now, NULL,
+	if ( ( rc = x509_validate_chain ( chain, now, NULL,
 					  validator->root ) ) == 0 ) {
 		DBGC ( validator, "VALIDATOR %p \"%s\" validated\n",
 		       validator, validator_name ( validator ) );
 		validator_finished ( validator, 0 );
 		return;
 	}
+	DBGC ( validator, "VALIDATOR %p \"%s\" not yet valid: %s\n",
+	       validator, validator_name ( validator ), strerror ( rc ) );
 
-	/* If there is a certificate that could be validated using
-	 * OCSP, try it.
+	/* Record as the most relevant error, if no more relevant
+	 * error has already been recorded.
 	 */
-	list_for_each_entry ( link, &validator->chain->links, list ) {
-		cert = issuer;
-		issuer = link->cert;
-		if ( ! cert )
-			continue;
-		if ( ! x509_is_valid ( issuer, validator->root ) )
-			continue;
-		/* The issuer is valid, but this certificate is not
-		 * yet valid.  If OCSP is applicable, start it.
-		 */
-		if ( ocsp_required ( cert ) ) {
-			/* Start OCSP */
-			if ( ( rc = validator_start_ocsp ( validator, cert,
-							   issuer ) ) != 0 ) {
-				validator_finished ( validator, rc );
-				return;
-			}
-			return;
-		}
-		/* Otherwise, this is a permanent failure */
-		validator_finished ( validator, rc );
-		return;
+	if ( validator->rc == 0 )
+		validator->rc = rc;
+
+	/* Find the first valid link in the chain, if any
+	 *
+	 * There is no point in attempting OCSP or cross-signed
+	 * certificate downloads for certificates after the first
+	 * valid link in the chain, since they cannot make a
+	 * difference to the overall validation of the chain.
+	 */
+	prev = NULL;
+	list_for_each_entry ( link, &chain->links, list ) {
+
+		/* Dump link information (for debugging) */
+		DBGC ( validator, "VALIDATOR %p \"%s\" has link ",
+		       validator, validator_name ( validator ) );
+		DBGC ( validator, "\"%s\"%s%s%s%s%s\n",
+		       x509_name ( link->cert ),
+		       ( ocsp_required ( link->cert ) ? " [NEEDOCSP]" : "" ),
+		       ( ( link->flags & X509_LINK_FL_OCSPED ) ?
+			 " [OCSPED]" : "" ),
+		       ( ( link->flags & X509_LINK_FL_CROSSED ) ?
+			 " [CROSSED]" : "" ),
+		       ( x509_is_self_signed ( link->cert ) ? " [SELF]" : "" ),
+		       ( x509_is_valid ( link->cert, validator->root ) ?
+			 " [VALID]" : "" ) );
+
+		/* Stop at first valid link */
+		if ( x509_is_valid ( link->cert, validator->root ) )
+			break;
+		prev = link;
 	}
 
-	/* If chain ends with a self-issued certificate, then there is
-	 * nothing more to do.
+	/* If this link is the issuer for a certificate that is
+	 * pending an OCSP check attempt, then start OCSP to validate
+	 * that certificate.
+	 *
+	 * If OCSP is not required for the issued certificate, or has
+	 * already been attempted, or if we were unable to start OCSP
+	 * for any reason, then proceed to attempting a cross-signed
+	 * certificate download (which may end up replacing this
+	 * issuer anyway).
 	 */
-	last = x509_last ( validator->chain );
-	if ( asn1_compare ( &last->issuer.raw, &last->subject.raw ) == 0 ) {
-		validator_finished ( validator, rc );
-		return;
+	if ( ( ! list_is_head_entry ( link, &chain->links, list ) ) &&
+	     ( ! ( link->flags & X509_LINK_FL_OCSPED ) ) &&
+	     ( prev != NULL ) && ocsp_required ( prev->cert ) ) {
+
+		/* Mark OCSP as attempted with this issuer */
+		link->flags |= X509_LINK_FL_OCSPED;
+
+		/* Start OCSP */
+		if ( ( rc = validator_start_ocsp ( validator, prev->cert,
+						   link->cert ) ) == 0 ) {
+			/* Sleep until OCSP is complete */
+			return;
+		}
 	}
 
-	/* Otherwise, try to download a suitable cross-signing
-	 * certificate.
+	/* Work back up the chain (starting from the already
+	 * identified first valid link, if any) to find a not-yet
+	 * valid certificate for which we could attempt to download a
+	 * cross-signed certificate chain.
 	 */
-	if ( ( rc = validator_start_download ( validator, last ) ) != 0 ) {
-		validator_finished ( validator, rc );
-		return;
+	list_for_each_entry_continue_reverse ( link, &chain->links, list ) {
+		cert = link->cert;
+
+		/* Sanity check */
+		assert ( ! x509_is_valid ( cert, validator->root ) );
+
+		/* Skip self-signed certificates (cannot be cross-signed) */
+		if ( x509_is_self_signed ( cert ) )
+			continue;
+
+		/* Skip previously attempted cross-signed downloads */
+		if ( link->flags & X509_LINK_FL_CROSSED )
+			continue;
+
+		/* Mark cross-signed certificate download as attempted */
+		link->flags |= X509_LINK_FL_CROSSED;
+
+		/* Start cross-signed certificate download */
+		if ( ( rc = validator_start_download ( validator,
+						       link ) ) == 0 ) {
+			/* Sleep until download is complete */
+			return;
+		}
 	}
+
+	/* Nothing more to try: fail the validation */
+	validator_finished ( validator, validator->rc );
 }
 
 /** Certificate validator process descriptor */
diff --git a/src/tests/aes_test.c b/src/tests/aes_test.c
index be119c8d..46a052a2 100644
--- a/src/tests/aes_test.c
+++ b/src/tests/aes_test.c
@@ -63,11 +63,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 	      0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, 0x2d, 0x98, 0x10,	\
 	      0xa3, 0x09, 0x14, 0xdf, 0xf4 )
 
-/** Dummy initialisation vector used for NIST ECB-mode test vectors */
-#define AES_IV_NIST_DUMMY						\
-	IV ( 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,	\
-	     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 )
-
 /** Initialisation vector used for NIST CBC-mode test vectors */
 #define AES_IV_NIST_CBC							\
 	IV ( 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,	\
@@ -86,7 +81,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** AES-128-ECB (same test as AES-128-Core) */
 CIPHER_TEST ( aes_128_ecb, &aes_ecb_algorithm,
-	AES_KEY_NIST_128, AES_IV_NIST_DUMMY, ADDITIONAL(), AES_PLAINTEXT_NIST,
+	AES_KEY_NIST_128, IV(), ADDITIONAL(), AES_PLAINTEXT_NIST,
 	CIPHERTEXT ( 0x3a, 0xd7, 0x7b, 0xb4, 0x0d, 0x7a, 0x36, 0x60,
 		     0xa8, 0x9e, 0xca, 0xf3, 0x24, 0x66, 0xef, 0x97,
 		     0xf5, 0xd3, 0xd5, 0x85, 0x03, 0xb9, 0x69, 0x9d,
@@ -110,7 +105,7 @@ CIPHER_TEST ( aes_128_cbc, &aes_cbc_algorithm,
 
 /** AES-192-ECB (same test as AES-192-Core) */
 CIPHER_TEST ( aes_192_ecb, &aes_ecb_algorithm,
-	AES_KEY_NIST_192, AES_IV_NIST_DUMMY, ADDITIONAL(), AES_PLAINTEXT_NIST,
+	AES_KEY_NIST_192, IV(), ADDITIONAL(), AES_PLAINTEXT_NIST,
 	CIPHERTEXT ( 0xbd, 0x33, 0x4f, 0x1d, 0x6e, 0x45, 0xf2, 0x5f,
 		     0xf7, 0x12, 0xa2, 0x14, 0x57, 0x1f, 0xa5, 0xcc,
 		     0x97, 0x41, 0x04, 0x84, 0x6d, 0x0a, 0xd3, 0xad,
@@ -134,7 +129,7 @@ CIPHER_TEST ( aes_192_cbc, &aes_cbc_algorithm,
 
 /** AES-256-ECB (same test as AES-256-Core) */
 CIPHER_TEST ( aes_256_ecb, &aes_ecb_algorithm,
-	AES_KEY_NIST_256, AES_IV_NIST_DUMMY, ADDITIONAL(), AES_PLAINTEXT_NIST,
+	AES_KEY_NIST_256, IV(), ADDITIONAL(), AES_PLAINTEXT_NIST,
 	CIPHERTEXT ( 0xf3, 0xee, 0xd1, 0xbd, 0xb5, 0xd2, 0xa0, 0x3c,
 		     0x06, 0x4b, 0x5a, 0x7e, 0x3d, 0xb1, 0x81, 0xf8,
 		     0x59, 0x1c, 0xcb, 0x10, 0xd4, 0x10, 0xed, 0x26,
diff --git a/src/tests/bigint_test.c b/src/tests/bigint_test.c
index 8d40c318..76aca105 100644
--- a/src/tests/bigint_test.c
+++ b/src/tests/bigint_test.c
@@ -149,16 +149,38 @@ void bigint_shrink_sample ( const bigint_element_t *source0,
 	bigint_shrink ( source, dest );
 }
 
+void bigint_copy_sample ( const bigint_element_t *source0,
+			  bigint_element_t *dest0, unsigned int size ) {
+	const bigint_t ( size ) *source __attribute__ (( may_alias ))
+		= ( ( const void * ) source0 );
+	bigint_t ( size ) *dest __attribute__ (( may_alias ))
+		= ( ( void * ) dest0 );
+
+	bigint_copy ( source, dest );
+}
+
+void bigint_swap_sample ( bigint_element_t *first0, bigint_element_t *second0,
+			  unsigned int size, int swap ) {
+	bigint_t ( size ) *first __attribute__ (( may_alias ))
+		= ( ( void * ) first0 );
+	bigint_t ( size ) *second __attribute__ (( may_alias ))
+		= ( ( void * ) second0 );
+
+	bigint_swap ( first, second, swap );
+}
+
 void bigint_multiply_sample ( const bigint_element_t *multiplicand0,
+			      unsigned int multiplicand_size,
 			      const bigint_element_t *multiplier0,
-			      bigint_element_t *result0,
-			      unsigned int size ) {
-	const bigint_t ( size ) *multiplicand __attribute__ (( may_alias ))
-		= ( ( const void * ) multiplicand0 );
-	const bigint_t ( size ) *multiplier __attribute__ (( may_alias ))
-		= ( ( const void * ) multiplier0 );
-	bigint_t ( size * 2 ) *result __attribute__ (( may_alias ))
-		= ( ( void * ) result0 );
+			      unsigned int multiplier_size,
+			      bigint_element_t *result0 ) {
+	unsigned int result_size = ( multiplicand_size + multiplier_size );
+	const bigint_t ( multiplicand_size ) __attribute__ (( may_alias ))
+		*multiplicand = ( ( const void * ) multiplicand0 );
+	const bigint_t ( multiplier_size ) __attribute__ (( may_alias ))
+		*multiplier = ( ( const void * ) multiplier0 );
+	bigint_t ( result_size ) __attribute__ (( may_alias ))
+		*result = ( ( void * ) result0 );
 
 	bigint_multiply ( multiplicand, multiplier, result );
 }
@@ -419,6 +441,42 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 	} while ( 0 )
 
 /**
+ * Report result of big integer swap test
+ *
+ * @v first		Big integer to be conditionally swapped
+ * @v second		Big integer to be conditionally swapped
+ */
+#define bigint_swap_ok( first, second ) do {				\
+	static const uint8_t first_raw[] = first;			\
+	static const uint8_t second_raw[] = second;			\
+	uint8_t temp[ sizeof ( first_raw ) ];				\
+	unsigned int size = bigint_required_size ( sizeof ( temp) );	\
+	bigint_t ( size ) first_temp;					\
+	bigint_t ( size ) second_temp;					\
+	{} /* Fix emacs alignment */					\
+									\
+	assert ( sizeof ( first_raw ) == sizeof ( temp ) );		\
+	assert ( sizeof ( second_raw ) == sizeof ( temp ) );		\
+	bigint_init ( &first_temp, first_raw, sizeof ( first_raw ) );	\
+	bigint_init ( &second_temp, second_raw, sizeof ( second_raw ) );\
+	bigint_swap ( &first_temp, &second_temp, 0 );			\
+	bigint_done ( &first_temp, temp, sizeof ( temp ) );		\
+	ok ( memcmp ( temp, first_raw, sizeof ( temp ) ) == 0 );	\
+	bigint_done ( &second_temp, temp, sizeof ( temp ) );		\
+	ok ( memcmp ( temp, second_raw, sizeof ( temp ) ) == 0 );	\
+	bigint_swap ( &first_temp, &second_temp, 1 );			\
+	bigint_done ( &first_temp, temp, sizeof ( temp ) );		\
+	ok ( memcmp ( temp, second_raw, sizeof ( temp ) ) == 0 );	\
+	bigint_done ( &second_temp, temp, sizeof ( temp ) );		\
+	ok ( memcmp ( temp, first_raw, sizeof ( temp ) ) == 0 );	\
+	bigint_swap ( &first_temp, &second_temp, 1 );			\
+	bigint_done ( &first_temp, temp, sizeof ( temp ) );		\
+	ok ( memcmp ( temp, first_raw, sizeof ( temp ) ) == 0 );	\
+	bigint_done ( &second_temp, temp, sizeof ( temp ) );		\
+	ok ( memcmp ( temp, second_raw, sizeof ( temp ) ) == 0 );	\
+	} while ( 0 )
+
+/**
  * Report result of big integer multiplication test
  *
  * @v multiplicand	Big integer to be multiplied
@@ -430,17 +488,18 @@ void bigint_mod_exp_sample ( const bigint_element_t *base0,
 	static const uint8_t multiplier_raw[] = multiplier;		\
 	static const uint8_t expected_raw[] = expected;			\
 	uint8_t result_raw[ sizeof ( expected_raw ) ];			\
-	unsigned int size =						\
+	unsigned int multiplicand_size =				\
 		bigint_required_size ( sizeof ( multiplicand_raw ) );	\
-	bigint_t ( size ) multiplicand_temp;				\
-	bigint_t ( size ) multiplier_temp;				\
-	bigint_t ( size * 2 ) result_temp;				\
+	unsigned int multiplier_size =					\
+		bigint_required_size ( sizeof ( multiplier_raw ) );	\
+	bigint_t ( multiplicand_size ) multiplicand_temp;		\
+	bigint_t ( multiplier_size ) multiplier_temp;			\
+	bigint_t ( multiplicand_size + multiplier_size ) result_temp;	\
 	{} /* Fix emacs alignment */					\
 									\
-	assert ( bigint_size ( &multiplier_temp ) ==			\
-		 bigint_size ( &multiplicand_temp ) );			\
 	assert ( bigint_size ( &result_temp ) ==			\
-		 ( 2 * bigint_size ( &multiplicand_temp ) ) );		\
+		 ( bigint_size ( &multiplicand_temp ) +			\
+		   bigint_size ( &multiplier_temp ) ) );		\
 	bigint_init ( &multiplicand_temp, multiplicand_raw,		\
 		      sizeof ( multiplicand_raw ) );			\
 	bigint_init ( &multiplier_temp, multiplier_raw,			\
@@ -655,6 +714,15 @@ static void bigint_test_exec ( void ) {
 	bigint_subtract_ok ( BIGINT ( 0xbb, 0x77, 0x32, 0x5a ),
 			     BIGINT ( 0x5a, 0xd5, 0xfe, 0x28 ),
 			     BIGINT ( 0x9f, 0x5e, 0xcb, 0xce ) );
+	bigint_subtract_ok ( BIGINT ( 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+				      0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+				      0xff, 0xff, 0xff, 0xff, 0xff, 0xff ),
+			     BIGINT ( 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				      0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				      0x00, 0x00, 0x00, 0x00, 0x00, 0x2a ),
+			     BIGINT ( 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+				      0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				      0x00, 0x00, 0x00, 0x00, 0x00, 0x2b ) );
 	bigint_subtract_ok ( BIGINT ( 0x7b, 0xaa, 0x16, 0xcf, 0x15, 0x87,
 				      0xe0, 0x4f, 0x2c, 0xa3, 0xec, 0x2f,
 				      0x46, 0xfb, 0x83, 0xc6, 0xe0, 0xee,
@@ -1360,6 +1428,14 @@ static void bigint_test_exec ( void ) {
 					 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
 					 0xff, 0xff ),
 				1024 );
+	bigint_swap_ok ( BIGINT ( 0x68, 0x65, 0x6c, 0x6c, 0x6f ),
+			 BIGINT ( 0x77, 0x6f, 0x72, 0x6c, 0x64 ) );
+	bigint_swap_ok ( BIGINT ( 0xc8, 0x1c, 0x31, 0xd7, 0x13, 0x69, 0x47,
+				  0x32, 0xb0, 0x0a, 0xf7, 0x2d, 0xb9, 0xc3,
+				  0x35, 0x96 ),
+			 BIGINT ( 0x8b, 0x1d, 0x8f, 0x21, 0x76, 0x16, 0x4c,
+				  0xf8, 0xb2, 0x63, 0xed, 0x89, 0x5e, 0x6b,
+				  0x35, 0x7c ) );
 	bigint_multiply_ok ( BIGINT ( 0xf0 ),
 			     BIGINT ( 0xeb ),
 			     BIGINT ( 0xdc, 0x50 ) );
@@ -1373,6 +1449,12 @@ static void bigint_test_exec ( void ) {
 			     BIGINT ( 0x67, 0x3c, 0x5a, 0x16 ),
 			     BIGINT ( 0x3c, 0xdb, 0x7f, 0xae, 0x12, 0x7e,
 				      0xef, 0x16 ) );
+	bigint_multiply_ok ( BIGINT ( 0x39, 0x1f, 0xc8, 0x6a ),
+			     BIGINT ( 0xba, 0x39, 0x4a, 0xb8, 0xac, 0xb3,
+				      0x4f, 0x64, 0x28, 0x46, 0xa6, 0x99 ),
+			     BIGINT ( 0x29, 0x8d, 0xe0, 0x5d, 0x08, 0xea,
+				      0x0d, 0xc7, 0x82, 0x5d, 0xba, 0x96,
+				      0x1c, 0xef, 0x83, 0x5a ) );
 	bigint_multiply_ok ( BIGINT ( 0xe8, 0x08, 0x0b, 0xe9, 0x29, 0x36,
 				      0xea, 0x51, 0x1d, 0x75, 0x1a, 0xd5,
 				      0xba, 0xc6, 0xa0, 0xf3, 0x48, 0x5c,
diff --git a/src/tests/des_test.c b/src/tests/des_test.c
new file mode 100644
index 00000000..ffafbd81
--- /dev/null
+++ b/src/tests/des_test.c
@@ -0,0 +1,898 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * DES tests
+ *
+ * These test vectors are originally provided by NBS (the precursor of
+ * NIST) in SP 500-20, downloadable as a scan of the typewritten
+ * original from:
+ *
+ * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nbsspecialpublication500-20e1980.pdf
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <assert.h>
+#include <ipxe/des.h>
+#include <ipxe/test.h>
+#include "cipher_test.h"
+
+/** Define a DES 64-bit test value */
+#define DES_VALUE(value) {						\
+	( ( ( ( uint64_t ) (value) ) >> 56 ) & 0xff ),			\
+	( ( ( ( uint64_t ) (value) ) >> 48 ) & 0xff ),			\
+	( ( ( ( uint64_t ) (value) ) >> 40 ) & 0xff ),			\
+	( ( ( ( uint64_t ) (value) ) >> 32 ) & 0xff ),			\
+	( ( ( ( uint64_t ) (value) ) >> 24 ) & 0xff ),			\
+	( ( ( ( uint64_t ) (value) ) >> 16 ) & 0xff ),			\
+	( ( ( ( uint64_t ) (value) ) >>  8 ) & 0xff ),			\
+	( ( ( ( uint64_t ) (value) ) >>  0 ) & 0xff )			\
+	}
+
+/** Define a DES test */
+#define DES_TEST( name, key, plaintext, ciphertext )			\
+	CIPHER_TEST ( name, &des_ecb_algorithm,	DES_VALUE ( key ),	\
+		      IV(), ADDITIONAL(), DES_VALUE ( plaintext ),	\
+		      DES_VALUE ( ciphertext ), AUTH() )
+
+/* Sample round outputs (page 9) */
+DES_TEST ( des_round_sample,
+	   0x10316e028c8f3b4a, 0x0000000000000000, 0x82dcbafbdeab6602 );
+
+/* Test 1: Initial permutation and expansion tests
+ *
+ * "Set Key=0 and encrypt the 64-bit data vectors e[i]: i=1,...,64"
+ *
+ * Appendix B, page 28 ("IP and E test")
+ */
+DES_TEST ( des_test1_1,
+	   0x0101010101010101, 0x8000000000000000, 0x95f8a5e5dd31d900 );
+DES_TEST ( des_test1_2,
+	   0x0101010101010101, 0x4000000000000000, 0xdd7f121ca5015619 );
+DES_TEST ( des_test1_3,
+	   0x0101010101010101, 0x2000000000000000, 0x2e8653104f3834ea );
+DES_TEST ( des_test1_4,
+	   0x0101010101010101, 0x1000000000000000, 0x4bd388ff6cd81d4f );
+DES_TEST ( des_test1_5,
+	   0x0101010101010101, 0x0800000000000000, 0x20b9e767b2fb1456 );
+DES_TEST ( des_test1_6,
+	   0x0101010101010101, 0x0400000000000000, 0x55579380d77138ef );
+DES_TEST ( des_test1_7,
+	   0x0101010101010101, 0x0200000000000000, 0x6cc5defaaf04512f );
+DES_TEST ( des_test1_8,
+	   0x0101010101010101, 0x0100000000000000, 0x0d9f279ba5d87260 );
+DES_TEST ( des_test1_9,
+	   0x0101010101010101, 0x0080000000000000, 0xd9031b0271bd5a0a );
+DES_TEST ( des_test1_10,
+	   0x0101010101010101, 0x0040000000000000, 0x424250b37c3dd951 );
+DES_TEST ( des_test1_11,
+	   0x0101010101010101, 0x0020000000000000, 0xb8061b7ecd9a21e5 );
+DES_TEST ( des_test1_12,
+	   0x0101010101010101, 0x0010000000000000, 0xf15d0f286b65bd28 );
+DES_TEST ( des_test1_13,
+	   0x0101010101010101, 0x0008000000000000, 0xadd0cc8d6e5deba1 );
+DES_TEST ( des_test1_14,
+	   0x0101010101010101, 0x0004000000000000, 0xe6d5f82752ad63d1 );
+DES_TEST ( des_test1_15,
+	   0x0101010101010101, 0x0002000000000000, 0xecbfe3bd3f591a5e );
+DES_TEST ( des_test1_16,
+	   0x0101010101010101, 0x0001000000000000, 0xf356834379d165cd );
+DES_TEST ( des_test1_17,
+	   0x0101010101010101, 0x0000800000000000, 0x2b9f982f20037fa9 );
+DES_TEST ( des_test1_18,
+	   0x0101010101010101, 0x0000400000000000, 0x889de068a16f0be6 );
+DES_TEST ( des_test1_19,
+	   0x0101010101010101, 0x0000200000000000, 0xe19e275d846a1298 );
+DES_TEST ( des_test1_20,
+	   0x0101010101010101, 0x0000100000000000, 0x329a8ed523d71aec );
+DES_TEST ( des_test1_21,
+	   0x0101010101010101, 0x0000080000000000, 0xe7fce22557d23c97 );
+DES_TEST ( des_test1_22,
+	   0x0101010101010101, 0x0000040000000000, 0x12a9f5817ff2d65d );
+DES_TEST ( des_test1_23,
+	   0x0101010101010101, 0x0000020000000000, 0xa484c3ad38dc9c19 );
+DES_TEST ( des_test1_24,
+	   0x0101010101010101, 0x0000010000000000, 0xfbe00a8a1ef8ad72 );
+DES_TEST ( des_test1_25,
+	   0x0101010101010101, 0x0000008000000000, 0x750d079407521363 );
+DES_TEST ( des_test1_26,
+	   0x0101010101010101, 0x0000004000000000, 0x64feed9c724c2faf );
+DES_TEST ( des_test1_27,
+	   0x0101010101010101, 0x0000002000000000, 0xf02b263b328e2b60 );
+DES_TEST ( des_test1_28,
+	   0x0101010101010101, 0x0000001000000000, 0x9d64555a9a10b852 );
+DES_TEST ( des_test1_29,
+	   0x0101010101010101, 0x0000000800000000, 0xd106ff0bed5255d7 );
+DES_TEST ( des_test1_30,
+	   0x0101010101010101, 0x0000000400000000, 0xe1652c6b138c64a5 );
+DES_TEST ( des_test1_31,
+	   0x0101010101010101, 0x0000000200000000, 0xe428581186ec8f46 );
+DES_TEST ( des_test1_32,
+	   0x0101010101010101, 0x0000000100000000, 0xaeb5f5ede22d1a36 );
+DES_TEST ( des_test1_33,
+	   0x0101010101010101, 0x0000000080000000, 0xe943d7568aec0c5c );
+DES_TEST ( des_test1_34,
+	   0x0101010101010101, 0x0000000040000000, 0xdf98c8276f54b04b );
+DES_TEST ( des_test1_35,
+	   0x0101010101010101, 0x0000000020000000, 0xb160e4680f6c696f );
+DES_TEST ( des_test1_36,
+	   0x0101010101010101, 0x0000000010000000, 0xfa0752b07d9c4ab8 );
+DES_TEST ( des_test1_37,
+	   0x0101010101010101, 0x0000000008000000, 0xca3a2b036dbc8502 );
+DES_TEST ( des_test1_38,
+	   0x0101010101010101, 0x0000000004000000, 0x5e0905517bb59bcf );
+DES_TEST ( des_test1_39,
+	   0x0101010101010101, 0x0000000002000000, 0x814eeb3b91d90726 );
+DES_TEST ( des_test1_40,
+	   0x0101010101010101, 0x0000000001000000, 0x4d49db1532919c9f );
+DES_TEST ( des_test1_41,
+	   0x0101010101010101, 0x0000000000800000, 0x25eb5fc3f8cf0621 );
+DES_TEST ( des_test1_42,
+	   0x0101010101010101, 0x0000000000400000, 0xab6a20c0620d1c6f );
+DES_TEST ( des_test1_43,
+	   0x0101010101010101, 0x0000000000200000, 0x79e90dbc98f92cca );
+DES_TEST ( des_test1_44,
+	   0x0101010101010101, 0x0000000000100000, 0x866ecedd8072bb0e );
+DES_TEST ( des_test1_45,
+	   0x0101010101010101, 0x0000000000080000, 0x8b54536f2f3e64a8 );
+DES_TEST ( des_test1_46,
+	   0x0101010101010101, 0x0000000000040000, 0xea51d3975595b86b );
+DES_TEST ( des_test1_47,
+	   0x0101010101010101, 0x0000000000020000, 0xcaffc6ac4542de31 );
+DES_TEST ( des_test1_48,
+	   0x0101010101010101, 0x0000000000010000, 0x8dd45a2ddf90796c );
+DES_TEST ( des_test1_49,
+	   0x0101010101010101, 0x0000000000008000, 0x1029d55e880ec2d0 );
+DES_TEST ( des_test1_50,
+	   0x0101010101010101, 0x0000000000004000, 0x5d86cb23639dbea9 );
+DES_TEST ( des_test1_51,
+	   0x0101010101010101, 0x0000000000002000, 0x1d1ca853ae7c0c5f );
+DES_TEST ( des_test1_52,
+	   0x0101010101010101, 0x0000000000001000, 0xce332329248f3228 );
+DES_TEST ( des_test1_53,
+	   0x0101010101010101, 0x0000000000000800, 0x8405d1abe24fb942 );
+DES_TEST ( des_test1_54,
+	   0x0101010101010101, 0x0000000000000400, 0xe643d78090ca4207 );
+DES_TEST ( des_test1_55,
+	   0x0101010101010101, 0x0000000000000200, 0x48221b9937748a23 );
+DES_TEST ( des_test1_56,
+	   0x0101010101010101, 0x0000000000000100, 0xdd7c0bbd61fafd54 );
+DES_TEST ( des_test1_57,
+	   0x0101010101010101, 0x0000000000000080, 0x2fbc291a570db5c4 );
+DES_TEST ( des_test1_58,
+	   0x0101010101010101, 0x0000000000000040, 0xe07c30d7e4e26e12 );
+DES_TEST ( des_test1_59,
+	   0x0101010101010101, 0x0000000000000020, 0x0953e2258e8e90a1 );
+DES_TEST ( des_test1_60,
+	   0x0101010101010101, 0x0000000000000010, 0x5b711bc4ceebf2ee );
+DES_TEST ( des_test1_61,
+	   0x0101010101010101, 0x0000000000000008, 0xcc083f1e6d9e85f6 );
+DES_TEST ( des_test1_62,
+	   0x0101010101010101, 0x0000000000000004, 0xd2fd8867d50d2dfe );
+DES_TEST ( des_test1_63,
+	   0x0101010101010101, 0x0000000000000002, 0x06e7ea22ce92708f );
+DES_TEST ( des_test1_64,
+	   0x0101010101010101, 0x0000000000000001, 0x166b40b44aba4bd6 );
+
+/* Test 2: Inverse permutation and expansion tests
+ *
+ * "Set Key=0 and encrypt the results c[i] obtained in Test 1"
+ *
+ * Appendix B, page 28 ("IP and E test")
+ */
+DES_TEST ( des_test2_1,
+	   0x0101010101010101, 0x95f8a5e5dd31d900, 0x8000000000000000 );
+DES_TEST ( des_test2_2,
+	   0x0101010101010101, 0xdd7f121ca5015619, 0x4000000000000000 );
+DES_TEST ( des_test2_3,
+	   0x0101010101010101, 0x2e8653104f3834ea, 0x2000000000000000 );
+DES_TEST ( des_test2_4,
+	   0x0101010101010101, 0x4bd388ff6cd81d4f, 0x1000000000000000 );
+DES_TEST ( des_test2_5,
+	   0x0101010101010101, 0x20b9e767b2fb1456, 0x0800000000000000 );
+DES_TEST ( des_test2_6,
+	   0x0101010101010101, 0x55579380d77138ef, 0x0400000000000000 );
+DES_TEST ( des_test2_7,
+	   0x0101010101010101, 0x6cc5defaaf04512f, 0x0200000000000000 );
+DES_TEST ( des_test2_8,
+	   0x0101010101010101, 0x0d9f279ba5d87260, 0x0100000000000000 );
+DES_TEST ( des_test2_9,
+	   0x0101010101010101, 0xd9031b0271bd5a0a, 0x0080000000000000 );
+DES_TEST ( des_test2_10,
+	   0x0101010101010101, 0x424250b37c3dd951, 0x0040000000000000 );
+DES_TEST ( des_test2_11,
+	   0x0101010101010101, 0xb8061b7ecd9a21e5, 0x0020000000000000 );
+DES_TEST ( des_test2_12,
+	   0x0101010101010101, 0xf15d0f286b65bd28, 0x0010000000000000 );
+DES_TEST ( des_test2_13,
+	   0x0101010101010101, 0xadd0cc8d6e5deba1, 0x0008000000000000 );
+DES_TEST ( des_test2_14,
+	   0x0101010101010101, 0xe6d5f82752ad63d1, 0x0004000000000000 );
+DES_TEST ( des_test2_15,
+	   0x0101010101010101, 0xecbfe3bd3f591a5e, 0x0002000000000000 );
+DES_TEST ( des_test2_16,
+	   0x0101010101010101, 0xf356834379d165cd, 0x0001000000000000 );
+DES_TEST ( des_test2_17,
+	   0x0101010101010101, 0x2b9f982f20037fa9, 0x0000800000000000 );
+DES_TEST ( des_test2_18,
+	   0x0101010101010101, 0x889de068a16f0be6, 0x0000400000000000 );
+DES_TEST ( des_test2_19,
+	   0x0101010101010101, 0xe19e275d846a1298, 0x0000200000000000 );
+DES_TEST ( des_test2_20,
+	   0x0101010101010101, 0x329a8ed523d71aec, 0x0000100000000000 );
+DES_TEST ( des_test2_21,
+	   0x0101010101010101, 0xe7fce22557d23c97, 0x0000080000000000 );
+DES_TEST ( des_test2_22,
+	   0x0101010101010101, 0x12a9f5817ff2d65d, 0x0000040000000000 );
+DES_TEST ( des_test2_23,
+	   0x0101010101010101, 0xa484c3ad38dc9c19, 0x0000020000000000 );
+DES_TEST ( des_test2_24,
+	   0x0101010101010101, 0xfbe00a8a1ef8ad72, 0x0000010000000000 );
+DES_TEST ( des_test2_25,
+	   0x0101010101010101, 0x750d079407521363, 0x0000008000000000 );
+DES_TEST ( des_test2_26,
+	   0x0101010101010101, 0x64feed9c724c2faf, 0x0000004000000000 );
+DES_TEST ( des_test2_27,
+	   0x0101010101010101, 0xf02b263b328e2b60, 0x0000002000000000 );
+DES_TEST ( des_test2_28,
+	   0x0101010101010101, 0x9d64555a9a10b852, 0x0000001000000000 );
+DES_TEST ( des_test2_29,
+	   0x0101010101010101, 0xd106ff0bed5255d7, 0x0000000800000000 );
+DES_TEST ( des_test2_30,
+	   0x0101010101010101, 0xe1652c6b138c64a5, 0x0000000400000000 );
+DES_TEST ( des_test2_31,
+	   0x0101010101010101, 0xe428581186ec8f46, 0x0000000200000000 );
+DES_TEST ( des_test2_32,
+	   0x0101010101010101, 0xaeb5f5ede22d1a36, 0x0000000100000000 );
+DES_TEST ( des_test2_33,
+	   0x0101010101010101, 0xe943d7568aec0c5c, 0x0000000080000000 );
+DES_TEST ( des_test2_34,
+	   0x0101010101010101, 0xdf98c8276f54b04b, 0x0000000040000000 );
+DES_TEST ( des_test2_35,
+	   0x0101010101010101, 0xb160e4680f6c696f, 0x0000000020000000 );
+DES_TEST ( des_test2_36,
+	   0x0101010101010101, 0xfa0752b07d9c4ab8, 0x0000000010000000 );
+DES_TEST ( des_test2_37,
+	   0x0101010101010101, 0xca3a2b036dbc8502, 0x0000000008000000 );
+DES_TEST ( des_test2_38,
+	   0x0101010101010101, 0x5e0905517bb59bcf, 0x0000000004000000 );
+DES_TEST ( des_test2_39,
+	   0x0101010101010101, 0x814eeb3b91d90726, 0x0000000002000000 );
+DES_TEST ( des_test2_40,
+	   0x0101010101010101, 0x4d49db1532919c9f, 0x0000000001000000 );
+DES_TEST ( des_test2_41,
+	   0x0101010101010101, 0x25eb5fc3f8cf0621, 0x0000000000800000 );
+DES_TEST ( des_test2_42,
+	   0x0101010101010101, 0xab6a20c0620d1c6f, 0x0000000000400000 );
+DES_TEST ( des_test2_43,
+	   0x0101010101010101, 0x79e90dbc98f92cca, 0x0000000000200000 );
+DES_TEST ( des_test2_44,
+	   0x0101010101010101, 0x866ecedd8072bb0e, 0x0000000000100000 );
+DES_TEST ( des_test2_45,
+	   0x0101010101010101, 0x8b54536f2f3e64a8, 0x0000000000080000 );
+DES_TEST ( des_test2_46,
+	   0x0101010101010101, 0xea51d3975595b86b, 0x0000000000040000 );
+DES_TEST ( des_test2_47,
+	   0x0101010101010101, 0xcaffc6ac4542de31, 0x0000000000020000 );
+DES_TEST ( des_test2_48,
+	   0x0101010101010101, 0x8dd45a2ddf90796c, 0x0000000000010000 );
+DES_TEST ( des_test2_49,
+	   0x0101010101010101, 0x1029d55e880ec2d0, 0x0000000000008000 );
+DES_TEST ( des_test2_50,
+	   0x0101010101010101, 0x5d86cb23639dbea9, 0x0000000000004000 );
+DES_TEST ( des_test2_51,
+	   0x0101010101010101, 0x1d1ca853ae7c0c5f, 0x0000000000002000 );
+DES_TEST ( des_test2_52,
+	   0x0101010101010101, 0xce332329248f3228, 0x0000000000001000 );
+DES_TEST ( des_test2_53,
+	   0x0101010101010101, 0x8405d1abe24fb942, 0x0000000000000800 );
+DES_TEST ( des_test2_54,
+	   0x0101010101010101, 0xe643d78090ca4207, 0x0000000000000400 );
+DES_TEST ( des_test2_55,
+	   0x0101010101010101, 0x48221b9937748a23, 0x0000000000000200 );
+DES_TEST ( des_test2_56,
+	   0x0101010101010101, 0xdd7c0bbd61fafd54, 0x0000000000000100 );
+DES_TEST ( des_test2_57,
+	   0x0101010101010101, 0x2fbc291a570db5c4, 0x0000000000000080 );
+DES_TEST ( des_test2_58,
+	   0x0101010101010101, 0xe07c30d7e4e26e12, 0x0000000000000040 );
+DES_TEST ( des_test2_59,
+	   0x0101010101010101, 0x0953e2258e8e90a1, 0x0000000000000020 );
+DES_TEST ( des_test2_60,
+	   0x0101010101010101, 0x5b711bc4ceebf2ee, 0x0000000000000010 );
+DES_TEST ( des_test2_61,
+	   0x0101010101010101, 0xcc083f1e6d9e85f6, 0x0000000000000008 );
+DES_TEST ( des_test2_62,
+	   0x0101010101010101, 0xd2fd8867d50d2dfe, 0x0000000000000004 );
+DES_TEST ( des_test2_63,
+	   0x0101010101010101, 0x06e7ea22ce92708f, 0x0000000000000002 );
+DES_TEST ( des_test2_64,
+	   0x0101010101010101, 0x166b40b44aba4bd6, 0x0000000000000001 );
+
+/* Test 3: Data permutation tests
+ *
+ * "Set the plaintext to zero and process the 32 keys in PTEST"
+ *
+ * Appendix B, page 32 ("PTEST")
+ */
+DES_TEST ( des_test3_1,
+	   0x1046913489980131, 0x0000000000000000, 0x88d55e54f54c97b4 );
+DES_TEST ( des_test3_2,
+	   0x1007103489988020, 0x0000000000000000, 0x0c0cc00c83ea48fd );
+DES_TEST ( des_test3_3,
+	   0x10071034c8980120, 0x0000000000000000, 0x83bc8ef3a6570183 );
+DES_TEST ( des_test3_4,
+	   0x1046103489988020, 0x0000000000000000, 0xdf725dcad94ea2e9 );
+DES_TEST ( des_test3_5,
+	   0x1086911519190101, 0x0000000000000000, 0xe652b53b550be8b0 );
+DES_TEST ( des_test3_6,
+	   0x1086911519580101, 0x0000000000000000, 0xaf527120c485cbb0 );
+DES_TEST ( des_test3_7,
+	   0x5107b01519580101, 0x0000000000000000, 0x0f04ce393db926d5 );
+DES_TEST ( des_test3_8,
+	   0x1007b01519190101, 0x0000000000000000, 0xc9f00ffc74079067 );
+DES_TEST ( des_test3_9,
+	   0x3107915498080101, 0x0000000000000000, 0x7cfd82a593252b4e );
+DES_TEST ( des_test3_10,
+	   0x3107919498080101, 0x0000000000000000, 0xcb49a2f9e91363e3 );
+DES_TEST ( des_test3_11,
+	   0x10079115b9080140, 0x0000000000000000, 0x00b588be70d23f56 );
+DES_TEST ( des_test3_12,
+	   0x3107911598080140, 0x0000000000000000, 0x406a9a6ab43399ae );
+DES_TEST ( des_test3_13,
+	   0x1007d01589980101, 0x0000000000000000, 0x6cb773611dca9ada );
+DES_TEST ( des_test3_14,
+	   0x9107911589980101, 0x0000000000000000, 0x67fd21c17dbb5d70 );
+DES_TEST ( des_test3_15,
+	   0x9107d01589190101, 0x0000000000000000, 0x9592cb4110430787 );
+DES_TEST ( des_test3_16,
+	   0x1007d01598980120, 0x0000000000000000, 0xa6b7ff68a318ddd3 );
+DES_TEST ( des_test3_17,
+	   0x1007940498190101, 0x0000000000000000, 0x4d102196c914ca16 );
+DES_TEST ( des_test3_18,
+	   0x0107910491190401, 0x0000000000000000, 0x2dfa9f4573594965 );
+DES_TEST ( des_test3_19,
+	   0x0107910491190101, 0x0000000000000000, 0xb46604816c0e0774 );
+DES_TEST ( des_test3_20,
+	   0x0107940491190401, 0x0000000000000000, 0x6e7e6221a4f34e87 );
+DES_TEST ( des_test3_21,
+	   0x19079210981a0101, 0x0000000000000000, 0xaa85e74643233199 );
+DES_TEST ( des_test3_22,
+	   0x1007911998190801, 0x0000000000000000, 0x2e5a19db4d1962d6 );
+DES_TEST ( des_test3_23,
+	   0x10079119981a0801, 0x0000000000000000, 0x23a866a809d30894 );
+DES_TEST ( des_test3_24,
+	   0x1007921098190101, 0x0000000000000000, 0xd812d961f017d320 );
+DES_TEST ( des_test3_25,
+	   0x100791159819010b, 0x0000000000000000, 0x055605816e58608f );
+DES_TEST ( des_test3_26,
+	   0x1004801598190101, 0x0000000000000000, 0xabd88e8b1b7716f1 );
+DES_TEST ( des_test3_27,
+	   0x1004801598190102, 0x0000000000000000, 0x537ac95be69da1e1 );
+DES_TEST ( des_test3_28,
+	   0x1004801598190108, 0x0000000000000000, 0xaed0f6ae3c25cdd8 );
+DES_TEST ( des_test3_29,
+	   0x1002911498100104, 0x0000000000000000, 0xb3e35a5ee53e7b8d );
+DES_TEST ( des_test3_30,
+	   0x1002911598190104, 0x0000000000000000, 0x61c79c71921a2ef8 );
+DES_TEST ( des_test3_31,
+	   0x1002911598100201, 0x0000000000000000, 0xe2f5728f0995013c );
+DES_TEST ( des_test3_32,
+	   0x1002911698100101, 0x0000000000000000, 0x1aeac39a61f0a464 );
+
+/* Test 4: Key permutation tests
+ *
+ * "Set Data=0 and use the keys e[i]: i=1,...,64 ignoring i=8,16,...,64"
+ *
+ * Test 4 part 1 is the forward direction as described above.  Test 4
+ * part 2 ("set data=c[i] from part 1 ... then decipher") is carried
+ * out for us automatically, since CIPHER_TEST() performs both
+ * encryption and decryption tests.
+ *
+ * Appendix B, page 30 ("PC1 and PC2 test")
+ */
+DES_TEST ( des_test4_1,
+	   0x8001010101010101, 0x0000000000000000, 0x95a8d72813daa94d );
+DES_TEST ( des_test4_2,
+	   0x4001010101010101, 0x0000000000000000, 0x0eec1487dd8c26d5 );
+DES_TEST ( des_test4_3,
+	   0x2001010101010101, 0x0000000000000000, 0x7ad16ffb79c45926 );
+DES_TEST ( des_test4_4,
+	   0x1001010101010101, 0x0000000000000000, 0xd3746294ca6a6cf3 );
+DES_TEST ( des_test4_5,
+	   0x0801010101010101, 0x0000000000000000, 0x809f5f873c1fd761 );
+DES_TEST ( des_test4_6,
+	   0x0401010101010101, 0x0000000000000000, 0xc02faffec989d1fc );
+DES_TEST ( des_test4_7,
+	   0x0201010101010101, 0x0000000000000000, 0x4615aa1d33e72f10 );
+DES_TEST ( des_test4_8,
+	   0x0180010101010101, 0x0000000000000000, 0x2055123350c00858 );
+DES_TEST ( des_test4_9,
+	   0x0140010101010101, 0x0000000000000000, 0xdf3b99d6577397c8 );
+DES_TEST ( des_test4_10,
+	   0x0120010101010101, 0x0000000000000000, 0x31fe17369b5288c9 );
+DES_TEST ( des_test4_11,
+	   0x0110010101010101, 0x0000000000000000, 0xdfdd3cc64dae1642 );
+DES_TEST ( des_test4_12,
+	   0x0108010101010101, 0x0000000000000000, 0x178c83ce2b399d94 );
+DES_TEST ( des_test4_13,
+	   0x0104010101010101, 0x0000000000000000, 0x50f636324a9b7f80 );
+DES_TEST ( des_test4_14,
+	   0x0102010101010101, 0x0000000000000000, 0xa8468ee3bc18f06d );
+DES_TEST ( des_test4_15,
+	   0x0101800101010101, 0x0000000000000000, 0xa2dc9e92fd3cde92 );
+DES_TEST ( des_test4_16,
+	   0x0101400101010101, 0x0000000000000000, 0xcac09f797d031287 );
+DES_TEST ( des_test4_17,
+	   0x0101200101010101, 0x0000000000000000, 0x90ba680b22aeb525 );
+DES_TEST ( des_test4_18,
+	   0x0101100101010101, 0x0000000000000000, 0xce7a24f350e280b6 );
+DES_TEST ( des_test4_19,
+	   0x0101080101010101, 0x0000000000000000, 0x882bff0aa01a0b87 );
+DES_TEST ( des_test4_20,
+	   0x0101040101010101, 0x0000000000000000, 0x25610288924511c2 );
+DES_TEST ( des_test4_21,
+	   0x0101020101010101, 0x0000000000000000, 0xc71516c29c75d170 );
+DES_TEST ( des_test4_22,
+	   0x0101018001010101, 0x0000000000000000, 0x5199c29a52c9f059 );
+DES_TEST ( des_test4_23,
+	   0x0101014001010101, 0x0000000000000000, 0xc22f0a294a71f29f );
+DES_TEST ( des_test4_24,
+	   0x0101012001010101, 0x0000000000000000, 0xee371483714c02ea );
+DES_TEST ( des_test4_25,
+	   0x0101011001010101, 0x0000000000000000, 0xa81fbd448f9e522f );
+DES_TEST ( des_test4_26,
+	   0x0101010801010101, 0x0000000000000000, 0x4f644c92e192dfed );
+DES_TEST ( des_test4_27,
+	   0x0101010401010101, 0x0000000000000000, 0x1afa9a66a6df92ae );
+DES_TEST ( des_test4_28,
+	   0x0101010201010101, 0x0000000000000000, 0xb3c1cc715cb879d8 );
+DES_TEST ( des_test4_29,
+	   0x0101010180010101, 0x0000000000000000, 0x19d032e64ab0bd8b );
+DES_TEST ( des_test4_30,
+	   0x0101010140010101, 0x0000000000000000, 0x3cfaa7a7dc8720dc );
+DES_TEST ( des_test4_31,
+	   0x0101010120010101, 0x0000000000000000, 0xb7265f7f447ac6f3 );
+DES_TEST ( des_test4_32,
+	   0x0101010110010101, 0x0000000000000000, 0x9db73b3c0d163f54 );
+DES_TEST ( des_test4_33,
+	   0x0101010108010101, 0x0000000000000000, 0x8181b65babf4a975 );
+DES_TEST ( des_test4_34,
+	   0x0101010104010101, 0x0000000000000000, 0x93c9b64042eaa240 );
+DES_TEST ( des_test4_35,
+	   0x0101010102010101, 0x0000000000000000, 0x5570530829705592 );
+DES_TEST ( des_test4_36,
+	   0x0101010101800101, 0x0000000000000000, 0x8638809e878787a0 );
+DES_TEST ( des_test4_37,
+	   0x0101010101400101, 0x0000000000000000, 0x41b9a79af79ac208 );
+DES_TEST ( des_test4_38,
+	   0x0101010101200101, 0x0000000000000000, 0x7a9be42f2009a892 );
+DES_TEST ( des_test4_39,
+	   0x0101010101100101, 0x0000000000000000, 0x29038d56ba6d2745 );
+DES_TEST ( des_test4_40,
+	   0x0101010101080101, 0x0000000000000000, 0x5495c6abf1e5df51 );
+DES_TEST ( des_test4_41,
+	   0x0101010101040101, 0x0000000000000000, 0xae13dbd561488933 );
+DES_TEST ( des_test4_42,
+	   0x0101010101020101, 0x0000000000000000, 0x024d1ffa8904e389 );
+DES_TEST ( des_test4_43,
+	   0x0101010101018001, 0x0000000000000000, 0xd1399712f99bf02e );
+DES_TEST ( des_test4_44,
+	   0x0101010101014001, 0x0000000000000000, 0x14c1d7c1cffec79e );
+DES_TEST ( des_test4_45,
+	   0x0101010101012001, 0x0000000000000000, 0x1de5279dae3bed6f );
+DES_TEST ( des_test4_46,
+	   0x0101010101011001, 0x0000000000000000, 0xe941a33f85501303 );
+DES_TEST ( des_test4_47,
+	   0x0101010101010801, 0x0000000000000000, 0xda99dbbc9a03f379 );
+DES_TEST ( des_test4_48,
+	   0x0101010101010401, 0x0000000000000000, 0xb7fc92f91d8e92e9 );
+DES_TEST ( des_test4_49,
+	   0x0101010101010201, 0x0000000000000000, 0xae8e5caa3ca04e85 );
+DES_TEST ( des_test4_50,
+	   0x0101010101010180, 0x0000000000000000, 0x9cc62df43b6eed74 );
+DES_TEST ( des_test4_51,
+	   0x0101010101010140, 0x0000000000000000, 0xd863dbb5c59a91a0 );
+DES_TEST ( des_test4_52,
+	   0x0101010101010120, 0x0000000000000000, 0xa1ab2190545b91d7 );
+DES_TEST ( des_test4_53,
+	   0x0101010101010110, 0x0000000000000000, 0x0875041e64c570f7 );
+DES_TEST ( des_test4_54,
+	   0x0101010101010108, 0x0000000000000000, 0x5a594528bebef1cc );
+DES_TEST ( des_test4_55,
+	   0x0101010101010104, 0x0000000000000000, 0xfcdb3291de21f0c0 );
+DES_TEST ( des_test4_56,
+	   0x0101010101010102, 0x0000000000000000, 0x869efd7f9f265a09 );
+
+/* Test 5: S-box tests
+ *
+ * "Set Data and Key equal to the inputs defined in the Substitution
+ * Table test"
+ *
+ * Appendix B, page 33 ("19 key data pairs which exercise every S-box entry")
+ */
+DES_TEST ( des_test5_1,
+	   0x7ca110454a1a6e57, 0x01a1d6d039776742, 0x690f5b0d9a26939b );
+DES_TEST ( des_test5_2,
+	   0x0131d9619dc1376e, 0x5cd54ca83def57da, 0x7a389d10354bd271 );
+DES_TEST ( des_test5_3,
+	   0x07a1133e4a0b2686, 0x0248d43806f67172, 0x868ebb51cab4599a );
+DES_TEST ( des_test5_4,
+	   0x3849674c2602319e, 0x51454b582ddf440a, 0x7178876e01f19b2a );
+DES_TEST ( des_test5_5,
+	   0x04b915ba43feb5b6, 0x42fd443059577fa2, 0xaf37fb421f8c4095 );
+DES_TEST ( des_test5_6,
+	   0x0113b970fd34f2ce, 0x059b5e0851cf143a, 0x86a560f10ec6d85b );
+DES_TEST ( des_test5_7,
+	   0x0170f175468fb5e6, 0x0756d8e0774761d2, 0x0cd3da020021dc09 );
+DES_TEST ( des_test5_8,
+	   0x43297fad38e373fe, 0x762514b829bf486a, 0xea676b2cb7db2b7a );
+DES_TEST ( des_test5_9,
+	   0x07a7137045da2a16, 0x3bdd119049372802, 0xdfd64a815caf1a0f );
+DES_TEST ( des_test5_10,
+	   0x04689104c2fd3b2f, 0x26955f6835af609a, 0x5c513c9c4886c088 );
+DES_TEST ( des_test5_11,
+	   0x37d06bb516cb7546, 0x164d5e404f275232, 0x0a2aeeae3ff4ab77 );
+DES_TEST ( des_test5_12,
+	   0x1f08260d1ac2465e, 0x6b056e18759f5cca, 0xef1bf03e5dfa575a );
+DES_TEST ( des_test5_13,
+	   0x584023641aba6176, 0x004bd6ef09176062, 0x88bf0db6d70dee56 );
+DES_TEST ( des_test5_14,
+	   0x025816164629b007, 0x480d39006ee762f2, 0xa1f9915541020b56 );
+DES_TEST ( des_test5_15,
+	   0x49793ebc79b3258f, 0x437540c8698f3cfa, 0x6fbf1cafcffd0556 );
+DES_TEST ( des_test5_16,
+	   0x4fb05e1515ab73a7, 0x072d43a077075292, 0x2f22e49bab7ca1ac );
+DES_TEST ( des_test5_17,
+	   0x49e95d6d4ca229bf, 0x02fe55778117f12a, 0x5a6b612cc26cce4a );
+DES_TEST ( des_test5_18,
+	   0x018310dc409b26d6, 0x1d9d5c5018f728c2, 0x5f4c038ed12b2e41 );
+DES_TEST ( des_test5_19,
+	   0x1c587f1c13924fef, 0x305532286d6f295a, 0x63fac0d034d9f793 );
+
+/* Unofficial tests
+ *
+ * The official tests are all exactly one block in length.  Add some
+ * multi-block tests (generated in Python).
+ */
+CIPHER_TEST ( des_unofficial_ecb, &des_ecb_algorithm,
+	      KEY ( 0x6e, 0x6f, 0x70, 0x61, 0x72, 0x69, 0x74, 0x79 ),
+	      IV(), ADDITIONAL(),
+	      PLAINTEXT ( 0x53, 0x6f, 0x20, 0x63, 0x75, 0x74, 0x65, 0x20,
+			  0x74, 0x6f, 0x20, 0x73, 0x65, 0x65, 0x20, 0x61,
+			  0x20, 0x73, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69,
+			  0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x77,
+			  0x72, 0x69, 0x74, 0x74, 0x65, 0x6e, 0x20, 0x6f,
+			  0x6e, 0x20, 0x61, 0x6e, 0x20, 0x61, 0x63, 0x74,
+			  0x75, 0x61, 0x6c, 0x20, 0x74, 0x79, 0x70, 0x65,
+			  0x77, 0x72, 0x69, 0x74, 0x65, 0x72, 0x21, 0x21 ),
+	      CIPHERTEXT ( 0x1a, 0x02, 0x17, 0xcb, 0x93, 0xa3, 0xd2, 0xf2,
+			   0xf9, 0x45, 0x71, 0x1c, 0x33, 0xb1, 0x5c, 0xa4,
+			   0x8b, 0x6b, 0x11, 0x7a, 0x7c, 0x86, 0x7c, 0x7f,
+			   0x9f, 0x56, 0x61, 0x46, 0x7f, 0xa6, 0xae, 0xf1,
+			   0x49, 0xf7, 0x53, 0xe0, 0xbc, 0x15, 0x6a, 0x30,
+			   0xe7, 0xf8, 0xf3, 0x29, 0x11, 0xd8, 0x7d, 0x04,
+			   0x62, 0x5a, 0xaa, 0xa1, 0x89, 0x61, 0x4c, 0xf6,
+			   0x5a, 0x47, 0x3b, 0xc6, 0x04, 0x15, 0xce, 0xf6 ),
+	      AUTH() );
+CIPHER_TEST ( des_unofficial_cbc, &des_cbc_algorithm,
+	      KEY ( 0x6e, 0x6f, 0x70, 0x61, 0x72, 0x69, 0x74, 0x79 ),
+	      IV ( 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 ),
+	      ADDITIONAL(),
+	      PLAINTEXT ( 0x53, 0x6f, 0x20, 0x63, 0x75, 0x74, 0x65, 0x20,
+			  0x74, 0x6f, 0x20, 0x73, 0x65, 0x65, 0x20, 0x61,
+			  0x20, 0x73, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69,
+			  0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x77,
+			  0x72, 0x69, 0x74, 0x74, 0x65, 0x6e, 0x20, 0x6f,
+			  0x6e, 0x20, 0x61, 0x6e, 0x20, 0x61, 0x63, 0x74,
+			  0x75, 0x61, 0x6c, 0x20, 0x74, 0x79, 0x70, 0x65,
+			  0x77, 0x72, 0x69, 0x74, 0x65, 0x72, 0x21, 0x21 ),
+	      CIPHERTEXT ( 0x4c, 0x5f, 0x62, 0xfc, 0xf4, 0x93, 0x09, 0xb5,
+			   0x1d, 0x52, 0x25, 0xec, 0xc7, 0x42, 0x3c, 0x29,
+			   0x33, 0x67, 0xf5, 0xe9, 0xd6, 0x3c, 0x27, 0x5b,
+			   0x49, 0x69, 0xc5, 0xa9, 0x08, 0xa3, 0x14, 0x66,
+			   0x3c, 0x95, 0x33, 0x30, 0xcf, 0x3c, 0x7c, 0xaf,
+			   0xa3, 0xe4, 0xf8, 0x2e, 0xc3, 0x55, 0x57, 0x81,
+			   0x33, 0xd9, 0x90, 0xe2, 0x99, 0xdc, 0x32, 0x10,
+			   0x13, 0x21, 0xb6, 0xc1, 0x6b, 0x0f, 0x22, 0xa9 ),
+	      AUTH() );
+
+/**
+ * Perform DES self-test
+ *
+ */
+static void des_test_exec ( void ) {
+
+	/* Sample round outputs (page 9) */
+	cipher_ok ( &des_round_sample );
+
+	/* Test 1: Initial permutation and expansion tests */
+	cipher_ok ( &des_test1_1 );
+	cipher_ok ( &des_test1_2 );
+	cipher_ok ( &des_test1_3 );
+	cipher_ok ( &des_test1_4 );
+	cipher_ok ( &des_test1_5 );
+	cipher_ok ( &des_test1_6 );
+	cipher_ok ( &des_test1_7 );
+	cipher_ok ( &des_test1_8 );
+	cipher_ok ( &des_test1_9 );
+	cipher_ok ( &des_test1_10 );
+	cipher_ok ( &des_test1_11 );
+	cipher_ok ( &des_test1_12 );
+	cipher_ok ( &des_test1_13 );
+	cipher_ok ( &des_test1_14 );
+	cipher_ok ( &des_test1_15 );
+	cipher_ok ( &des_test1_16 );
+	cipher_ok ( &des_test1_17 );
+	cipher_ok ( &des_test1_18 );
+	cipher_ok ( &des_test1_19 );
+	cipher_ok ( &des_test1_20 );
+	cipher_ok ( &des_test1_21 );
+	cipher_ok ( &des_test1_22 );
+	cipher_ok ( &des_test1_23 );
+	cipher_ok ( &des_test1_24 );
+	cipher_ok ( &des_test1_25 );
+	cipher_ok ( &des_test1_26 );
+	cipher_ok ( &des_test1_27 );
+	cipher_ok ( &des_test1_28 );
+	cipher_ok ( &des_test1_29 );
+	cipher_ok ( &des_test1_30 );
+	cipher_ok ( &des_test1_31 );
+	cipher_ok ( &des_test1_32 );
+	cipher_ok ( &des_test1_33 );
+	cipher_ok ( &des_test1_34 );
+	cipher_ok ( &des_test1_35 );
+	cipher_ok ( &des_test1_36 );
+	cipher_ok ( &des_test1_37 );
+	cipher_ok ( &des_test1_38 );
+	cipher_ok ( &des_test1_39 );
+	cipher_ok ( &des_test1_40 );
+	cipher_ok ( &des_test1_41 );
+	cipher_ok ( &des_test1_42 );
+	cipher_ok ( &des_test1_43 );
+	cipher_ok ( &des_test1_44 );
+	cipher_ok ( &des_test1_45 );
+	cipher_ok ( &des_test1_46 );
+	cipher_ok ( &des_test1_47 );
+	cipher_ok ( &des_test1_48 );
+	cipher_ok ( &des_test1_49 );
+	cipher_ok ( &des_test1_50 );
+	cipher_ok ( &des_test1_51 );
+	cipher_ok ( &des_test1_52 );
+	cipher_ok ( &des_test1_53 );
+	cipher_ok ( &des_test1_54 );
+	cipher_ok ( &des_test1_55 );
+	cipher_ok ( &des_test1_56 );
+	cipher_ok ( &des_test1_57 );
+	cipher_ok ( &des_test1_58 );
+	cipher_ok ( &des_test1_59 );
+	cipher_ok ( &des_test1_60 );
+	cipher_ok ( &des_test1_61 );
+	cipher_ok ( &des_test1_62 );
+	cipher_ok ( &des_test1_63 );
+	cipher_ok ( &des_test1_64 );
+
+	/* Test 2: Inverse permutation and expansion tests */
+	cipher_ok ( &des_test2_1 );
+	cipher_ok ( &des_test2_2 );
+	cipher_ok ( &des_test2_3 );
+	cipher_ok ( &des_test2_4 );
+	cipher_ok ( &des_test2_5 );
+	cipher_ok ( &des_test2_6 );
+	cipher_ok ( &des_test2_7 );
+	cipher_ok ( &des_test2_8 );
+	cipher_ok ( &des_test2_9 );
+	cipher_ok ( &des_test2_10 );
+	cipher_ok ( &des_test2_11 );
+	cipher_ok ( &des_test2_12 );
+	cipher_ok ( &des_test2_13 );
+	cipher_ok ( &des_test2_14 );
+	cipher_ok ( &des_test2_15 );
+	cipher_ok ( &des_test2_16 );
+	cipher_ok ( &des_test2_17 );
+	cipher_ok ( &des_test2_18 );
+	cipher_ok ( &des_test2_19 );
+	cipher_ok ( &des_test2_20 );
+	cipher_ok ( &des_test2_21 );
+	cipher_ok ( &des_test2_22 );
+	cipher_ok ( &des_test2_23 );
+	cipher_ok ( &des_test2_24 );
+	cipher_ok ( &des_test2_25 );
+	cipher_ok ( &des_test2_26 );
+	cipher_ok ( &des_test2_27 );
+	cipher_ok ( &des_test2_28 );
+	cipher_ok ( &des_test2_29 );
+	cipher_ok ( &des_test2_30 );
+	cipher_ok ( &des_test2_31 );
+	cipher_ok ( &des_test2_32 );
+	cipher_ok ( &des_test2_33 );
+	cipher_ok ( &des_test2_34 );
+	cipher_ok ( &des_test2_35 );
+	cipher_ok ( &des_test2_36 );
+	cipher_ok ( &des_test2_37 );
+	cipher_ok ( &des_test2_38 );
+	cipher_ok ( &des_test2_39 );
+	cipher_ok ( &des_test2_40 );
+	cipher_ok ( &des_test2_41 );
+	cipher_ok ( &des_test2_42 );
+	cipher_ok ( &des_test2_43 );
+	cipher_ok ( &des_test2_44 );
+	cipher_ok ( &des_test2_45 );
+	cipher_ok ( &des_test2_46 );
+	cipher_ok ( &des_test2_47 );
+	cipher_ok ( &des_test2_48 );
+	cipher_ok ( &des_test2_49 );
+	cipher_ok ( &des_test2_50 );
+	cipher_ok ( &des_test2_51 );
+	cipher_ok ( &des_test2_52 );
+	cipher_ok ( &des_test2_53 );
+	cipher_ok ( &des_test2_54 );
+	cipher_ok ( &des_test2_55 );
+	cipher_ok ( &des_test2_56 );
+	cipher_ok ( &des_test2_57 );
+	cipher_ok ( &des_test2_58 );
+	cipher_ok ( &des_test2_59 );
+	cipher_ok ( &des_test2_60 );
+	cipher_ok ( &des_test2_61 );
+	cipher_ok ( &des_test2_62 );
+	cipher_ok ( &des_test2_63 );
+	cipher_ok ( &des_test2_64 );
+
+	/* Test 3: Data permutation tests */
+	cipher_ok ( &des_test3_1 );
+	cipher_ok ( &des_test3_2 );
+	cipher_ok ( &des_test3_3 );
+	cipher_ok ( &des_test3_4 );
+	cipher_ok ( &des_test3_5 );
+	cipher_ok ( &des_test3_6 );
+	cipher_ok ( &des_test3_7 );
+	cipher_ok ( &des_test3_8 );
+	cipher_ok ( &des_test3_9 );
+	cipher_ok ( &des_test3_10 );
+	cipher_ok ( &des_test3_11 );
+	cipher_ok ( &des_test3_12 );
+	cipher_ok ( &des_test3_13 );
+	cipher_ok ( &des_test3_14 );
+	cipher_ok ( &des_test3_15 );
+	cipher_ok ( &des_test3_16 );
+	cipher_ok ( &des_test3_17 );
+	cipher_ok ( &des_test3_18 );
+	cipher_ok ( &des_test3_19 );
+	cipher_ok ( &des_test3_20 );
+	cipher_ok ( &des_test3_21 );
+	cipher_ok ( &des_test3_22 );
+	cipher_ok ( &des_test3_23 );
+	cipher_ok ( &des_test3_24 );
+	cipher_ok ( &des_test3_25 );
+	cipher_ok ( &des_test3_26 );
+	cipher_ok ( &des_test3_27 );
+	cipher_ok ( &des_test3_28 );
+	cipher_ok ( &des_test3_29 );
+	cipher_ok ( &des_test3_30 );
+	cipher_ok ( &des_test3_31 );
+	cipher_ok ( &des_test3_32 );
+
+	/* Test 4: Key permutation tests */
+	cipher_ok ( &des_test4_1 );
+	cipher_ok ( &des_test4_2 );
+	cipher_ok ( &des_test4_3 );
+	cipher_ok ( &des_test4_4 );
+	cipher_ok ( &des_test4_5 );
+	cipher_ok ( &des_test4_6 );
+	cipher_ok ( &des_test4_7 );
+	cipher_ok ( &des_test4_8 );
+	cipher_ok ( &des_test4_9 );
+	cipher_ok ( &des_test4_10 );
+	cipher_ok ( &des_test4_11 );
+	cipher_ok ( &des_test4_12 );
+	cipher_ok ( &des_test4_13 );
+	cipher_ok ( &des_test4_14 );
+	cipher_ok ( &des_test4_15 );
+	cipher_ok ( &des_test4_16 );
+	cipher_ok ( &des_test4_17 );
+	cipher_ok ( &des_test4_18 );
+	cipher_ok ( &des_test4_19 );
+	cipher_ok ( &des_test4_20 );
+	cipher_ok ( &des_test4_21 );
+	cipher_ok ( &des_test4_22 );
+	cipher_ok ( &des_test4_23 );
+	cipher_ok ( &des_test4_24 );
+	cipher_ok ( &des_test4_25 );
+	cipher_ok ( &des_test4_26 );
+	cipher_ok ( &des_test4_27 );
+	cipher_ok ( &des_test4_28 );
+	cipher_ok ( &des_test4_29 );
+	cipher_ok ( &des_test4_30 );
+	cipher_ok ( &des_test4_31 );
+	cipher_ok ( &des_test4_32 );
+	cipher_ok ( &des_test4_33 );
+	cipher_ok ( &des_test4_34 );
+	cipher_ok ( &des_test4_35 );
+	cipher_ok ( &des_test4_36 );
+	cipher_ok ( &des_test4_37 );
+	cipher_ok ( &des_test4_38 );
+	cipher_ok ( &des_test4_39 );
+	cipher_ok ( &des_test4_40 );
+	cipher_ok ( &des_test4_41 );
+	cipher_ok ( &des_test4_42 );
+	cipher_ok ( &des_test4_43 );
+	cipher_ok ( &des_test4_44 );
+	cipher_ok ( &des_test4_45 );
+	cipher_ok ( &des_test4_46 );
+	cipher_ok ( &des_test4_47 );
+	cipher_ok ( &des_test4_48 );
+	cipher_ok ( &des_test4_49 );
+	cipher_ok ( &des_test4_50 );
+	cipher_ok ( &des_test4_51 );
+	cipher_ok ( &des_test4_52 );
+	cipher_ok ( &des_test4_53 );
+	cipher_ok ( &des_test4_54 );
+	cipher_ok ( &des_test4_55 );
+	cipher_ok ( &des_test4_56 );
+
+	/* Test 5: S-box tests */
+	cipher_ok ( &des_test5_1 );
+	cipher_ok ( &des_test5_2 );
+	cipher_ok ( &des_test5_3 );
+	cipher_ok ( &des_test5_4 );
+	cipher_ok ( &des_test5_5 );
+	cipher_ok ( &des_test5_6 );
+	cipher_ok ( &des_test5_7 );
+	cipher_ok ( &des_test5_8 );
+	cipher_ok ( &des_test5_9 );
+	cipher_ok ( &des_test5_10 );
+	cipher_ok ( &des_test5_11 );
+	cipher_ok ( &des_test5_12 );
+	cipher_ok ( &des_test5_13 );
+	cipher_ok ( &des_test5_14 );
+	cipher_ok ( &des_test5_15 );
+	cipher_ok ( &des_test5_16 );
+	cipher_ok ( &des_test5_17 );
+	cipher_ok ( &des_test5_18 );
+	cipher_ok ( &des_test5_19 );
+
+	/* Multi-block tests */
+	cipher_ok ( &des_unofficial_ecb );
+	cipher_ok ( &des_unofficial_cbc );
+
+	/* Speed tests */
+	DBG ( "DES-ECB encryption required %ld cycles per byte\n",
+	      cipher_cost_encrypt ( &des_ecb_algorithm, 8 ) );
+	DBG ( "DES-ECB decryption required %ld cycles per byte\n",
+	      cipher_cost_decrypt ( &des_ecb_algorithm, 8 ) );
+	DBG ( "DES-CBC encryption required %ld cycles per byte\n",
+	      cipher_cost_encrypt ( &des_cbc_algorithm, 8 ) );
+	DBG ( "DES-CBC decryption required %ld cycles per byte\n",
+	      cipher_cost_decrypt ( &des_cbc_algorithm, 8 ) );
+}
+
+/** DES self-test */
+struct self_test des_test __self_test = {
+	.name = "des",
+	.exec = des_test_exec,
+};
diff --git a/src/tests/list_test.c b/src/tests/list_test.c
index d5b5c65d..c24e8082 100644
--- a/src/tests/list_test.c
+++ b/src/tests/list_test.c
@@ -440,6 +440,22 @@ static void list_test_exec ( void ) {
 	ok ( list_is_first_entry ( &list_tests[3], list, list ) );
 	ok ( list_is_last_entry ( &list_tests[3], list, list ) );
 
+	/* Test list_is_head_entry() */
+	INIT_LIST_HEAD ( list );
+	list_add_tail ( &list_tests[1].list, list );
+	list_add_tail ( &list_tests[6].list, list );
+	list_add_tail ( &list_tests[8].list, list );
+	ok ( list_is_head_entry ( list_entry ( list, typeof ( *pos ), list ),
+				  list, list ) );
+	ok ( ! list_is_head_entry ( &list_tests[1], list, list ) );
+	ok ( ! list_is_head_entry ( &list_tests[6], list, list ) );
+	ok ( ! list_is_head_entry ( &list_tests[8], list, list ) );
+	list_for_each_entry ( pos, list, list ) {
+		ok ( list_contains_entry ( pos, list, list ) );
+		ok ( ! list_is_head_entry ( pos, list, list ) );
+	}
+	ok ( list_is_head_entry ( pos, list, list ) );
+
 	/* Test list_for_each() */
 	INIT_LIST_HEAD ( list );
 	list_add_tail ( &list_tests[6].list, list );
@@ -502,6 +518,38 @@ static void list_test_exec ( void ) {
 	list_iterate_entry_ok ( list_for_each_entry_continue_reverse, "",
 				pos, list, list );
 
+	/* Test list_for_each_entry_safe_continue() */
+	INIT_LIST_HEAD ( list );
+	list_add_tail ( &list_tests[9].list, list );
+	list_add_tail ( &list_tests[4].list, list );
+	list_add_tail ( &list_tests[2].list, list );
+	list_add_tail ( &list_tests[5].list, list );
+	list_add_tail ( &list_tests[7].list, list );
+	{
+		char *expecteds[] = { "94257", "9457", "947", "94" };
+		char **expected = expecteds;
+		pos = &list_tests[4];
+		list_for_each_entry_safe_continue ( pos, tmp, list, list ) {
+			list_contents_ok ( list, *expected );
+			list_del ( &pos->list );
+			expected++;
+			list_contents_ok ( list, *expected );
+		}
+	}
+	list_contents_ok ( list, "94" );
+	{
+		char *expecteds[] = { "94", "4", "" };
+		char **expected = expecteds;
+		ok ( pos == list_entry ( list, struct list_test, list ) );
+		list_for_each_entry_safe_continue ( pos, tmp, list, list ) {
+			list_contents_ok ( list, *expected );
+			list_del ( &pos->list );
+			expected++;
+			list_contents_ok ( list, *expected );
+		}
+	}
+	ok ( list_empty ( list ) );
+
 	/* Test list_contains() and list_contains_entry() */
 	INIT_LIST_HEAD ( list );
 	INIT_LIST_HEAD ( &list_tests[3].list );
diff --git a/src/tests/mschapv2_test.c b/src/tests/mschapv2_test.c
new file mode 100644
index 00000000..3d10ed18
--- /dev/null
+++ b/src/tests/mschapv2_test.c
@@ -0,0 +1,144 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * MS-CHAPv2 authentication self-tests
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <stdlib.h>
+#include <string.h>
+#include <ipxe/mschapv2.h>
+#include <ipxe/test.h>
+
+/** An MS-CHAPv2 test */
+struct mschapv2_test {
+	/** Username */
+	const char *username;
+	/** Password */
+	const char *password;
+	/** Authenticator challenge */
+	const struct mschapv2_challenge *challenge;
+	/** Peer challenge */
+	const struct mschapv2_challenge *peer;
+	/** Expected challenge response */
+	const struct mschapv2_response *response;
+	/** Expected authenticator response */
+	const struct mschapv2_auth *auth;
+};
+
+/** Define inline data */
+#define DATA(...) { __VA_ARGS__ }
+
+/** Define an MS-CHAPv2 test */
+#define MSCHAPV2_TEST( name, USERNAME, PASSWORD, CHALLENGE, PEER,	\
+		       RESPONSE, AUTH )					\
+	static const struct mschapv2_challenge name ## _challenge = {	\
+		.byte = CHALLENGE,					\
+	};								\
+	static const struct mschapv2_challenge name ## _peer = {	\
+		.byte = PEER,						\
+	};								\
+	static const union {						\
+		struct mschapv2_response response;			\
+		uint8_t byte[ sizeof ( struct mschapv2_response ) ];	\
+	} name ## _response = {						\
+		.byte = RESPONSE,					\
+	};								\
+	static const union {						\
+		struct mschapv2_auth auth;				\
+		uint8_t byte[ sizeof ( struct mschapv2_auth ) ];	\
+	} name ## _auth = {						\
+		.byte = AUTH,						\
+	};								\
+	static struct mschapv2_test name = {				\
+		.username = USERNAME,					\
+		.password = PASSWORD,					\
+		.challenge = &name ## _challenge,			\
+		.peer = &name ## _peer,					\
+		.response = &name ## _response.response,		\
+		.auth = &name ## _auth.auth,				\
+	};
+
+/** RFC 2759 section 9.2 test case */
+MSCHAPV2_TEST ( rfc2759_test,
+		"User", "clientPass",
+		DATA ( 0x5b, 0x5d, 0x7c, 0x7d, 0x7b, 0x3f, 0x2f, 0x3e,
+		       0x3c, 0x2c, 0x60, 0x21, 0x32, 0x26, 0x26, 0x28 ),
+		DATA ( 0x21, 0x40, 0x23, 0x24, 0x25, 0x5e, 0x26, 0x2a,
+		       0x28, 0x29, 0x5f, 0x2b, 0x3a, 0x33, 0x7c, 0x7e ),
+		DATA ( 0x21, 0x40, 0x23, 0x24, 0x25, 0x5e, 0x26, 0x2a,
+		       0x28, 0x29, 0x5f, 0x2b, 0x3a, 0x33, 0x7c, 0x7e,
+		       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		       0x82, 0x30, 0x9e, 0xcd, 0x8d, 0x70, 0x8b, 0x5e,
+		       0xa0, 0x8f, 0xaa, 0x39, 0x81, 0xcd, 0x83, 0x54,
+		       0x42, 0x33, 0x11, 0x4a, 0x3d, 0x85, 0xd6, 0xdf,
+		       0x00 ),
+		"S=407A5589115FD0D6209F510FE9C04566932CDA56" );
+
+/**
+ * Report an MS-CHAPv2 test result
+ *
+ * @v test		Authentication test
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void mschapv2_okx ( struct mschapv2_test *test,
+			   const char *file, unsigned int line ) {
+	struct mschapv2_response response;
+	struct mschapv2_auth auth;
+
+	/* Compute challenge response */
+	mschapv2_response ( test->username, test->password, test->challenge,
+			    test->peer, &response );
+	okx ( memcmp ( &response, test->response, sizeof ( response ) ) == 0,
+	      file, line );
+
+	/* Compute authenticator response */
+	mschapv2_auth ( test->username, test->password, test->challenge,
+			test->response, &auth );
+	okx ( memcmp ( &auth, test->auth, sizeof ( auth ) ) == 0, file, line );
+}
+#define mschapv2_ok( test )				\
+	mschapv2_okx ( test, __FILE__, __LINE__ )
+
+/**
+ * Perform MS-CHAPv2 self-test
+ *
+ */
+static void mschapv2_test_exec ( void ) {
+
+	mschapv2_ok ( &rfc2759_test );
+}
+
+/** MS-CHAPv2 self-test */
+struct self_test mschapv2_test __self_test = {
+	.name = "mschapv2",
+	.exec = mschapv2_test_exec,
+};
diff --git a/src/tests/settings_test.c b/src/tests/settings_test.c
index 5da7eb00..edd7b9d7 100644
--- a/src/tests/settings_test.c
+++ b/src/tests/settings_test.c
@@ -420,13 +420,21 @@ static void settings_test_exec ( void ) {
 		    RAW ( 0x80, 0x81, 0x82, 0x83, 0x84, 0x00, 0xff ),
 		    "gIGCg4QA/w==" );
 
-	/* "uuid" setting type (no store capability) */
+	/* "uuid" setting type */
+	storef_ok ( &test_settings, &test_uuid_setting,
+		    "36d22ed9-b64f-4fdb-941b-a54a0854f991",
+		    RAW ( 0x36, 0xd2, 0x2e, 0xd9, 0xb6, 0x4f, 0x4f, 0xdb, 0x94,
+			  0x1b, 0xa5, 0x4a, 0x08, 0x54, 0xf9, 0x91 ) );
+	storef_ok ( &test_settings, &test_guid_setting,
+		    "7ad4478f-c270-4601-a245-78598f25a984",
+		    RAW ( 0x8f, 0x47, 0xd4, 0x7a, 0x70, 0xc2, 0x01, 0x46, 0xa2,
+			  0x45, 0x78, 0x59, 0x8f, 0x25, 0xa9, 0x84 ) );
 	fetchf_ok ( &test_settings, &test_uuid_setting,
-		    RAW ( 0x1a, 0x6a, 0x74, 0x9d, 0x0e, 0xda, 0x46, 0x1a,0xa8,
+		    RAW ( 0x1a, 0x6a, 0x74, 0x9d, 0x0e, 0xda, 0x46, 0x1a, 0xa8,
 			  0x7a, 0x7c, 0xfe, 0x4f, 0xca, 0x4a, 0x57 ),
 		    "1a6a749d-0eda-461a-a87a-7cfe4fca4a57" );
 	fetchf_ok ( &test_settings, &test_guid_setting,
-		    RAW ( 0x1a, 0x6a, 0x74, 0x9d, 0x0e, 0xda, 0x46, 0x1a,0xa8,
+		    RAW ( 0x1a, 0x6a, 0x74, 0x9d, 0x0e, 0xda, 0x46, 0x1a, 0xa8,
 			  0x7a, 0x7c, 0xfe, 0x4f, 0xca, 0x4a, 0x57 ),
 		    "9d746a1a-da0e-1a46-a87a-7cfe4fca4a57" );
 
diff --git a/src/tests/tests.c b/src/tests/tests.c
index fbdf562c..cb296049 100644
--- a/src/tests/tests.c
+++ b/src/tests/tests.c
@@ -81,3 +81,7 @@ REQUIRE_OBJECT ( hmac_test );
 REQUIRE_OBJECT ( dhe_test );
 REQUIRE_OBJECT ( gcm_test );
 REQUIRE_OBJECT ( nap_test );
+REQUIRE_OBJECT ( x25519_test );
+REQUIRE_OBJECT ( des_test );
+REQUIRE_OBJECT ( mschapv2_test );
+REQUIRE_OBJECT ( uuid_test );
diff --git a/src/tests/uuid_test.c b/src/tests/uuid_test.c
new file mode 100644
index 00000000..42dc5264
--- /dev/null
+++ b/src/tests/uuid_test.c
@@ -0,0 +1,156 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * UUID tests
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <string.h>
+#include <byteswap.h>
+#include <ipxe/uuid.h>
+#include <ipxe/test.h>
+
+/** Define an inline UUID value */
+#define UUID( A, B, C, D, E0, E1, E2, E3, E4, E5 ) {		\
+		.a = htonl ( A ),				\
+		.b = htons ( B ),				\
+		.c = htons ( C ),				\
+		.d = htons ( D ),				\
+		.e = { E0, E1, E2, E3, E4, E5 },		\
+	}
+
+/**
+ * Report a uuid_ntoa() test result
+ *
+ * @v uuid		UUID
+ * @v text		Expected textual representation
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void uuid_ntoa_okx ( const union uuid *uuid, const char *text,
+			    const char *file, unsigned int line ) {
+	const char *actual;
+
+	/* Format address */
+	actual = uuid_ntoa ( uuid );
+	DBG ( "uuid_ntoa ( %08x-%04x-%04x-%04x-%02x%02x%02x%02x%02x%02x ) = "
+	      "\"%s\"\n", ntohl ( uuid->canonical.a ),
+	      ntohs ( uuid->canonical.b ), ntohs ( uuid->canonical.c ),
+	      ntohs ( uuid->canonical.d ), uuid->canonical.e[0],
+	      uuid->canonical.e[1], uuid->canonical.e[2], uuid->canonical.e[3],
+	      uuid->canonical.e[4], uuid->canonical.e[5], actual );
+	okx ( strcmp ( actual, text ) == 0, file, line );
+}
+#define uuid_ntoa_ok( value, text ) do {				\
+	static const union uuid uuid = {				\
+		.canonical = value,					\
+	};								\
+	uuid_ntoa_okx ( &uuid, text, __FILE__, __LINE__ );		\
+	} while ( 0 )
+
+/**
+ * Report a uuid_aton() test result
+ *
+ * @v text		Textual representation
+ * @v uuid		Expected UUID
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void uuid_aton_okx ( const char *text, const union uuid *uuid,
+			    const char *file, unsigned int line ) {
+	union uuid actual;
+
+	/* Parse address */
+	okx ( uuid_aton ( text, &actual ) == 0, file, line );
+	DBG ( "uuid_aton ( \"%s\" ) = %s\n", text, uuid_ntoa ( &actual ) );
+	okx ( memcmp ( &actual, uuid, sizeof ( actual ) ) == 0, file, line );
+};
+#define uuid_aton_ok( text, value ) do {				\
+	static const union uuid uuid = {				\
+		.canonical = value,					\
+	};								\
+	uuid_aton_okx ( text, &uuid, __FILE__, __LINE__ );		\
+	} while ( 0 )
+
+/**
+ * Report a uuid_aton() failure test result
+ *
+ * @v text		Textual representation
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void uuid_aton_fail_okx ( const char *text, const char *file,
+				 unsigned int line ) {
+	union uuid actual;
+
+	/* Attempt to parse address */
+	okx ( uuid_aton ( text, &actual ) != 0, file, line );
+}
+#define uuid_aton_fail_ok( text ) \
+	uuid_aton_fail_okx ( text, __FILE__, __LINE__ )
+
+/**
+ * Perform UUID self-tests
+ *
+ */
+static void uuid_test_exec ( void ) {
+
+	/* uuid_ntoa() tests */
+	uuid_ntoa_ok ( UUID ( 0x18725ca6, 0xd699, 0x4e4d, 0xb501,
+			      0xc3, 0x80, 0x91, 0xd2, 0xa4, 0x33 ),
+		       "18725ca6-d699-4e4d-b501-c38091d2a433" );
+	uuid_ntoa_ok ( UUID ( 0x1a969b23, 0xc7d5, 0x40fe, 0xb79a,
+			      0xc9, 0x2e, 0xa3, 0x4a ,0xb4, 0x5b ),
+		       "1a969b23-c7d5-40fe-b79a-c92ea34ab45b" );
+
+	/* uuid_aton() tests */
+	uuid_aton_ok ( "62b907a8-e1a7-460e-82f7-667d84270c84",
+		       UUID ( 0x62b907a8, 0xe1a7, 0x460e, 0x82f7,
+			      0x66, 0x7d, 0x84, 0x27, 0x0c, 0x84 ) );
+	uuid_aton_ok ( "F5D0349C-EF7C-4AD4-B40B-FC2E522A7327",
+		       UUID ( 0xf5d0349c, 0xef7c, 0x4ad4, 0xb40b,
+			      0xfc, 0x2e, 0x52, 0x2a, 0x73, 0x27 ) );
+	uuid_aton_ok ( "4edd80ff7b43465589a02b1e7cffa196",
+		       UUID ( 0x4edd80ff, 0x7b43, 0x4655, 0x89a0,
+			      0x2b, 0x1e, 0x7c, 0xff, 0xa1, 0x96 ) );
+
+	/* uuid_aton() failure tests */
+	uuid_aton_fail_ok ( "628d677b-cf38-471e-9ad9-c8a5d9220055b6" );
+	uuid_aton_fail_ok ( "5071ca26-fc5f-4580-887a-46d9a103e4" );
+	uuid_aton_fail_ok ( "453aee96:0fb5-4aeb-aecd-d060b2121218" );
+	uuid_aton_fail_ok ( "1ccb524a-b8b9-4b17-x5e2-7996867edc7d" );
+	uuid_aton_fail_ok ( "" );
+}
+
+/** UUID self-test */
+struct self_test uuid_test __self_test = {
+	.name = "uuid",
+	.exec = uuid_test_exec,
+};
diff --git a/src/tests/x25519_test.c b/src/tests/x25519_test.c
new file mode 100644
index 00000000..3dfbd339
--- /dev/null
+++ b/src/tests/x25519_test.c
@@ -0,0 +1,600 @@
+/*
+ * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * X25519 key exchange test
+ *
+ * Full key exchange test vectors are taken from RFC7748.
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <stdint.h>
+#include <string.h>
+#include <ipxe/x25519.h>
+#include <ipxe/test.h>
+
+/** Define inline multiplicand */
+#define MULTIPLICAND(...) { __VA_ARGS__ }
+
+/** Define inline multiplier */
+#define MULTIPLIER(...) { __VA_ARGS__ }
+
+/** Define inline invertend */
+#define INVERTEND(...) { __VA_ARGS__ }
+
+/** Define inline base point */
+#define BASE(...) { __VA_ARGS__ }
+
+/** Define inline scalar multiple */
+#define SCALAR(...) { __VA_ARGS__ }
+
+/** Define inline expected result */
+#define EXPECTED(...) { __VA_ARGS__ }
+
+/** An X25519 multiplication self-test */
+struct x25519_multiply_test {
+	/** Multiplicand */
+	const void *multiplicand;
+	/** Length of multiplicand */
+	size_t multiplicand_len;
+	/** Multiplier */
+	const void *multiplier;
+	/** Length of multiplier */
+	size_t multiplier_len;
+	/** Expected result */
+	const void *expected;
+	/** Length of expected result */
+	size_t expected_len;
+};
+
+/**
+ * Define an X25519 multiplication test
+ *
+ * @v name		Test name
+ * @v MULTIPLICAND	258-bit multiplicand
+ * @v MULTIPLIER	258-bit multiplier
+ * @v EXPECTED		255-bit expected result
+ * @ret test		X25519 multiplication test
+ */
+#define X25519_MULTIPLY_TEST( name, MULTIPLICAND, MULTIPLIER,		\
+			      EXPECTED )				\
+	static const uint8_t name ## _multiplicand[] = MULTIPLICAND;	\
+	static const uint8_t name ## _multiplier[] = MULTIPLIER;	\
+	static const uint8_t name ## _expected[] = EXPECTED;		\
+	static struct x25519_multiply_test name = {			\
+		.multiplicand = name ## _multiplicand,			\
+		.multiplicand_len = sizeof ( name ## _multiplicand ),	\
+		.multiplier = name ## _multiplier,			\
+		.multiplier_len = sizeof ( name ## _multiplier ),	\
+		.expected = name ## _expected,				\
+		.expected_len = sizeof ( name ## _expected ),		\
+	}
+
+/** An X25519 multiplicative inversion self-test */
+struct x25519_invert_test {
+	/** Invertend */
+	const void *invertend;
+	/** Length of invertend */
+	size_t invertend_len;
+	/** Expected result */
+	const void *expected;
+	/** Length of expected result */
+	size_t expected_len;
+};
+
+/**
+ * Define an X25519 multiplicative inversion test
+ *
+ * @v name		Test name
+ * @v INVERTEND		258-bit invertend
+ * @v EXPECTED		255-bit expected result
+ * @ret test		X25519 multiplicative inversion test
+ */
+#define X25519_INVERT_TEST( name, INVERTEND, EXPECTED )			\
+	static const uint8_t name ## _invertend[] = INVERTEND;		\
+	static const uint8_t name ## _expected[] = EXPECTED;		\
+	static struct x25519_invert_test name = {			\
+		.invertend = name ## _invertend,			\
+		.invertend_len = sizeof ( name ## _invertend ),		\
+		.expected = name ## _expected,				\
+		.expected_len = sizeof ( name ## _expected ),		\
+	}
+
+/** An X25519 key exchange self-test */
+struct x25519_key_test {
+	/** Base */
+	struct x25519_value base;
+	/** Scalar */
+	struct x25519_value scalar;
+	/** Expected result */
+	struct x25519_value expected;
+	/** Number of iterations */
+	unsigned int count;
+	/** Key exchange is expected to fail (i.e. produce all-zeroes) */
+	int fail;
+};
+
+/**
+ * Define an X25519 key exchange test
+ *
+ * @v name		Test name
+ * @v COUNT		Number of iterations
+ * @v FAIL		Expected failure status
+ * @v BASE		Base point
+ * @v SCALAR		Scalar multiple
+ * @v EXPECTED		Expected result
+ * @ret test		X25519 key exchange test
+ */
+#define X25519_KEY_TEST( name, COUNT, FAIL, BASE, SCALAR, EXPECTED )	\
+	static struct x25519_key_test name = {				\
+		.count = COUNT,						\
+		.fail = FAIL,						\
+		.base = { .raw = BASE },				\
+		.scalar = { .raw = SCALAR },				\
+		.expected = { .raw = EXPECTED },			\
+	}
+
+/**
+ * Report an X25519 multiplication test result
+ *
+ * @v test		X25519 multiplication test
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void x25519_multiply_okx ( struct x25519_multiply_test *test,
+				  const char *file, unsigned int line ) {
+	union x25519_oct258 multiplicand;
+	union x25519_oct258 multiplier;
+	union x25519_quad257 expected;
+	union x25519_quad257 actual;
+
+	/* Construct big integers */
+	bigint_init ( &multiplicand.value, test->multiplicand,
+		      test->multiplicand_len );
+	DBGC ( test, "X25519 multiplicand:\n" );
+	DBGC_HDA ( test, 0, &multiplicand, sizeof ( multiplicand ) );
+	bigint_init ( &multiplier.value, test->multiplier,
+		      test->multiplier_len );
+	DBGC ( test, "X25519 multiplier:\n" );
+	DBGC_HDA ( test, 0, &multiplier, sizeof ( multiplier ) );
+	bigint_init ( &expected.value, test->expected, test->expected_len );
+	DBGC ( test, "X25519 expected product:\n" );
+	DBGC_HDA ( test, 0, &expected, sizeof ( expected ) );
+
+	/* Perform multiplication */
+	x25519_multiply ( &multiplicand, &multiplier, &actual );
+
+	/* Reduce result to allow for comparison */
+	x25519_reduce ( &actual );
+	DBGC ( test, "X25519 actual product:\n" );
+	DBGC_HDA ( test, 0, &actual, sizeof ( actual ) );
+
+	/* Compare against expected result */
+	okx ( memcmp ( &actual, &expected, sizeof ( expected ) ) == 0,
+	      file, line );
+}
+#define x25519_multiply_ok( test ) \
+	x25519_multiply_okx ( test, __FILE__, __LINE__ )
+
+/**
+ * Report an X25519 multiplicative inversion test result
+ *
+ * @v test		X25519 multiplicative inversion test
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void x25519_invert_okx ( struct x25519_invert_test *test,
+				const char *file, unsigned int line ) {
+	static const uint8_t one[] = { 1 };
+	union x25519_oct258 invertend;
+	union x25519_quad257 expected;
+	union x25519_quad257 actual;
+	union x25519_quad257 product;
+	union x25519_quad257 identity;
+
+	/* Construct big integers */
+	bigint_init ( &invertend.value, test->invertend, test->invertend_len );
+	DBGC ( test, "X25519 invertend:\n" );
+	DBGC_HDA ( test, 0, &invertend, sizeof ( invertend ) );
+	bigint_init ( &expected.value, test->expected, test->expected_len );
+	DBGC ( test, "X25519 expected inverse:\n" );
+	DBGC_HDA ( test, 0, &expected, sizeof ( expected ) );
+	bigint_init ( &identity.value, one, sizeof ( one ) );
+
+	/* Perform inversion */
+	x25519_invert ( &invertend, &actual );
+
+	/* Multiply invertend by inverse */
+	x25519_multiply ( &invertend, &actual.oct258, &product );
+
+	/* Reduce results to allow for comparison */
+	x25519_reduce ( &actual );
+	DBGC ( test, "X25519 actual inverse:\n" );
+	DBGC_HDA ( test, 0, &actual, sizeof ( actual ) );
+	x25519_reduce ( &product );
+	DBGC ( test, "X25519 actual product:\n" );
+	DBGC_HDA ( test, 0, &product, sizeof ( product ) );
+
+	/* Compare against expected results */
+	okx ( memcmp ( &actual, &expected, sizeof ( expected ) ) == 0,
+	      file, line );
+	okx ( memcmp ( &product, &identity, sizeof ( identity ) ) == 0,
+	      file, line );
+}
+#define x25519_invert_ok( test ) \
+	x25519_invert_okx ( test, __FILE__, __LINE__ )
+
+/**
+ * Report an X25519 key exchange test result
+ *
+ * @v test		X25519 key exchange test
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void x25519_key_okx ( struct x25519_key_test *test,
+			     const char *file, unsigned int line ) {
+	struct x25519_value base;
+	struct x25519_value scalar;
+	struct x25519_value actual;
+	unsigned int i;
+	int rc;
+
+	/* Construct input values */
+	memcpy ( &base, &test->base, sizeof ( test->base ) );
+	memcpy ( &scalar, &test->scalar, sizeof ( test->scalar ) );
+	DBGC ( test, "X25519 base:\n" );
+	DBGC_HDA ( test, 0, &base, sizeof ( base ) );
+	DBGC ( test, "X25519 scalar:\n" );
+	DBGC_HDA ( test, 0, &scalar, sizeof ( scalar ) );
+	DBGC ( test, "X25519 expected result (x%d):\n", test->count );
+	DBGC_HDA ( test, 0, &test->expected, sizeof ( test->expected ) );
+
+	/* Calculate key */
+	for ( i = 0 ; i < test->count ; i++ ) {
+		rc = x25519_key ( &base, &scalar, &actual );
+		if ( test->fail ) {
+			okx ( rc != 0, file, line );
+		} else {
+			okx ( rc == 0, file, line );
+		}
+		memcpy ( &base, &scalar, sizeof ( base ) );
+		memcpy ( &scalar, &actual, sizeof ( scalar ) );
+	}
+	DBGC ( test, "X25519 actual result (x%d):\n", test->count );
+	DBGC_HDA ( test, 0, &actual, sizeof ( actual ) );
+
+	/* Compare against expected result */
+	okx ( memcmp ( &actual, &test->expected,
+		       sizeof ( test->expected ) ) == 0, file, line );
+}
+#define x25519_key_ok( test ) \
+	x25519_key_okx ( test, __FILE__, __LINE__ )
+
+/* Test multiplying small numbers */
+X25519_MULTIPLY_TEST ( multiply_small, MULTIPLICAND ( 6 ),
+		       MULTIPLIER ( 9 ), EXPECTED ( 6 * 9 ) );
+
+/* Test exact multiple of field prime */
+X25519_MULTIPLY_TEST ( multiply_k_p,
+	MULTIPLICAND ( 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		       0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		       0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		       0xff, 0xff, 0xff, 0xff, 0xed ),
+	MULTIPLIER ( 0x00, 0xe8, 0x0d, 0x83, 0xd4, 0xe9, 0x1e, 0xdd, 0x7a,
+		     0x45, 0x14, 0x87, 0xb7, 0xfc, 0x62, 0x54, 0x1f, 0xb2,
+		     0x97, 0x24, 0xde, 0xfa, 0xd3, 0xe7, 0x3e, 0x83, 0x93,
+		     0x60, 0xbc, 0x20, 0x97, 0x9b, 0x22 ),
+	EXPECTED ( 0x00 ) );
+
+/* 0x0223b8c1e9392456de3eb13b9046685257bdd640fb06671ad11c80317fa3b1799d *
+ * 0x006c031199972a846916419f828b9d2434e465e150bd9c66b3ad3c2d6d1a3d1fa7 =
+ * 0x1ba87e982f7c477616b4d5136ba54733e40081c1c2e27d864aa178ce893d1297 (mod p)
+ */
+X25519_MULTIPLY_TEST ( multiply_1,
+	MULTIPLICAND ( 0x02, 0x23, 0xb8, 0xc1, 0xe9, 0x39, 0x24, 0x56, 0xde,
+		       0x3e, 0xb1, 0x3b, 0x90, 0x46, 0x68, 0x52, 0x57, 0xbd,
+		       0xd6, 0x40, 0xfb, 0x06, 0x67, 0x1a, 0xd1, 0x1c, 0x80,
+		       0x31, 0x7f, 0xa3, 0xb1, 0x79, 0x9d ),
+	MULTIPLIER ( 0x00, 0x6c, 0x03, 0x11, 0x99, 0x97, 0x2a, 0x84, 0x69,
+		     0x16, 0x41, 0x9f, 0x82, 0x8b, 0x9d, 0x24, 0x34, 0xe4,
+		     0x65, 0xe1, 0x50, 0xbd, 0x9c, 0x66, 0xb3, 0xad, 0x3c,
+		     0x2d, 0x6d, 0x1a, 0x3d, 0x1f, 0xa7 ),
+	EXPECTED ( 0x1b, 0xa8, 0x7e, 0x98, 0x2f, 0x7c, 0x47, 0x76, 0x16, 0xb4,
+		   0xd5, 0x13, 0x6b, 0xa5, 0x47, 0x33, 0xe4, 0x00, 0x81, 0xc1,
+		   0xc2, 0xe2, 0x7d, 0x86, 0x4a, 0xa1, 0x78, 0xce, 0x89, 0x3d,
+		   0x12, 0x97 ) );
+
+/* 0x008fadc1a606cb0fb39a1de644815ef6d13b8faa1837f8a88b17fc695a07a0ca6e *
+ * 0x0196da1dac72ff5d2a386ecbe06b65a6a48b8148f6b38a088ca65ed389b74d0fb1 =
+ * 0x351f7bf75ef580249ed6f9ff3996463b0730a1d49b5d36b863e192591157e950 (mod p)
+ */
+X25519_MULTIPLY_TEST ( multiply_2,
+	MULTIPLICAND ( 0x00, 0x8f, 0xad, 0xc1, 0xa6, 0x06, 0xcb, 0x0f, 0xb3,
+		       0x9a, 0x1d, 0xe6, 0x44, 0x81, 0x5e, 0xf6, 0xd1, 0x3b,
+		       0x8f, 0xaa, 0x18, 0x37, 0xf8, 0xa8, 0x8b, 0x17, 0xfc,
+		       0x69, 0x5a, 0x07, 0xa0, 0xca, 0x6e ),
+	MULTIPLIER ( 0x01, 0x96, 0xda, 0x1d, 0xac, 0x72, 0xff, 0x5d, 0x2a,
+		     0x38, 0x6e, 0xcb, 0xe0, 0x6b, 0x65, 0xa6, 0xa4, 0x8b,
+		     0x81, 0x48, 0xf6, 0xb3, 0x8a, 0x08, 0x8c, 0xa6, 0x5e,
+		     0xd3, 0x89, 0xb7, 0x4d, 0x0f, 0xb1 ),
+	EXPECTED ( 0x35, 0x1f, 0x7b, 0xf7, 0x5e, 0xf5, 0x80, 0x24, 0x9e, 0xd6,
+		   0xf9, 0xff, 0x39, 0x96, 0x46, 0x3b, 0x07, 0x30, 0xa1, 0xd4,
+		   0x9b, 0x5d, 0x36, 0xb8, 0x63, 0xe1, 0x92, 0x59, 0x11, 0x57,
+		   0xe9, 0x50 ) );
+
+/* 0x016c307511b2b9437a28df6ec4ce4a2bbdc241330b01a9e71fde8a774bcf36d58b *
+ * 0x0117be31111a2a73ed562b0f79c37459eef50bea63371ecd7b27cd813047229389 =
+ * 0x6b43b5185965f8f0920f31ae1b2cefadd7b078fecf68dbeaa17b9c385b558329 (mod p)
+ */
+X25519_MULTIPLY_TEST ( multiply_3,
+	MULTIPLICAND ( 0x01, 0x6c, 0x30, 0x75, 0x11, 0xb2, 0xb9, 0x43, 0x7a,
+		       0x28, 0xdf, 0x6e, 0xc4, 0xce, 0x4a, 0x2b, 0xbd, 0xc2,
+		       0x41, 0x33, 0x0b, 0x01, 0xa9, 0xe7, 0x1f, 0xde, 0x8a,
+		       0x77, 0x4b, 0xcf, 0x36, 0xd5, 0x8b ),
+	MULTIPLIER ( 0x01, 0x17, 0xbe, 0x31, 0x11, 0x1a, 0x2a, 0x73, 0xed,
+		     0x56, 0x2b, 0x0f, 0x79, 0xc3, 0x74, 0x59, 0xee, 0xf5,
+		     0x0b, 0xea, 0x63, 0x37, 0x1e, 0xcd, 0x7b, 0x27, 0xcd,
+		     0x81, 0x30, 0x47, 0x22, 0x93, 0x89 ),
+	EXPECTED ( 0x6b, 0x43, 0xb5, 0x18, 0x59, 0x65, 0xf8, 0xf0, 0x92, 0x0f,
+		   0x31, 0xae, 0x1b, 0x2c, 0xef, 0xad, 0xd7, 0xb0, 0x78, 0xfe,
+		   0xcf, 0x68, 0xdb, 0xea, 0xa1, 0x7b, 0x9c, 0x38, 0x5b, 0x55,
+		   0x83, 0x29 ) );
+
+/* 0x020b1f9163ce9ff57f43b7a3a69a8dca03580d7b71d8f564135be6128e18c26797 *
+ * 0x018d5288f1142c3fe860e7a113ec1b8ca1f91e1d4c1ff49b7889463e85759cde66 =
+ * 0x28a77d3c8a14323d63b288dbd40315b3f192b8485d86a02cb87d3dfb7a0b5447 (mod p)
+ */
+X25519_MULTIPLY_TEST ( multiply_4,
+	MULTIPLICAND ( 0x02, 0x0b, 0x1f, 0x91, 0x63, 0xce, 0x9f, 0xf5, 0x7f,
+		       0x43, 0xb7, 0xa3, 0xa6, 0x9a, 0x8d, 0xca, 0x03, 0x58,
+		       0x0d, 0x7b, 0x71, 0xd8, 0xf5, 0x64, 0x13, 0x5b, 0xe6,
+		       0x12, 0x8e, 0x18, 0xc2, 0x67, 0x97 ),
+	MULTIPLIER ( 0x01, 0x8d, 0x52, 0x88, 0xf1, 0x14, 0x2c, 0x3f, 0xe8,
+		     0x60, 0xe7, 0xa1, 0x13, 0xec, 0x1b, 0x8c, 0xa1, 0xf9,
+		     0x1e, 0x1d, 0x4c, 0x1f, 0xf4, 0x9b, 0x78, 0x89, 0x46,
+		     0x3e, 0x85, 0x75, 0x9c, 0xde, 0x66 ),
+	EXPECTED ( 0x28, 0xa7, 0x7d, 0x3c, 0x8a, 0x14, 0x32, 0x3d, 0x63, 0xb2,
+		   0x88, 0xdb, 0xd4, 0x03, 0x15, 0xb3, 0xf1, 0x92, 0xb8, 0x48,
+		   0x5d, 0x86, 0xa0, 0x2c, 0xb8, 0x7d, 0x3d, 0xfb, 0x7a, 0x0b,
+		   0x54, 0x47 ) );
+
+/* 0x023139d32c93cd59bf5c941cf0dc98d2c1e2acf72f9e574f7aa0ee89aed453dd32 *
+ * 0x03146d3f31fc377a4c4a15544dc5e7ce8a3a578a8ea9488d990bbb259911ce5dd2 =
+ * 0x4bdb7a35c0a5182000aa67554741e88cfdf460a78c6fae07adf83d2f005d2767 (mod p)
+ */
+X25519_MULTIPLY_TEST ( multiply_5,
+	MULTIPLICAND ( 0x02, 0x31, 0x39, 0xd3, 0x2c, 0x93, 0xcd, 0x59, 0xbf,
+		       0x5c, 0x94, 0x1c, 0xf0, 0xdc, 0x98, 0xd2, 0xc1, 0xe2,
+		       0xac, 0xf7, 0x2f, 0x9e, 0x57, 0x4f, 0x7a, 0xa0, 0xee,
+		       0x89, 0xae, 0xd4, 0x53, 0xdd, 0x32 ),
+	MULTIPLIER ( 0x03, 0x14, 0x6d, 0x3f, 0x31, 0xfc, 0x37, 0x7a, 0x4c,
+		     0x4a, 0x15, 0x54, 0x4d, 0xc5, 0xe7, 0xce, 0x8a, 0x3a,
+		     0x57, 0x8a, 0x8e, 0xa9, 0x48, 0x8d, 0x99, 0x0b, 0xbb,
+		     0x25, 0x99, 0x11, 0xce, 0x5d, 0xd2 ),
+	EXPECTED ( 0x4b, 0xdb, 0x7a, 0x35, 0xc0, 0xa5, 0x18, 0x20, 0x00, 0xaa,
+		   0x67, 0x55, 0x47, 0x41, 0xe8, 0x8c, 0xfd, 0xf4, 0x60, 0xa7,
+		   0x8c, 0x6f, 0xae, 0x07, 0xad, 0xf8, 0x3d, 0x2f, 0x00, 0x5d,
+		   0x27, 0x67 ) );
+
+/* 0x01d58842dea2bc372f7412b29347294739614ff3d719db3ad0ddd1dfb23b982ef8 ^ -1 =
+ * 0x093ff51750809d181a9a5481c564e37cff618def8ec45f464b1a6e24f8b826bd (mod p)
+ */
+X25519_INVERT_TEST ( invert_1,
+	INVERTEND ( 0x01, 0xd5, 0x88, 0x42, 0xde, 0xa2, 0xbc, 0x37, 0x2f,
+		    0x74, 0x12, 0xb2, 0x93, 0x47, 0x29, 0x47, 0x39, 0x61,
+		    0x4f, 0xf3, 0xd7, 0x19, 0xdb, 0x3a, 0xd0, 0xdd, 0xd1,
+		    0xdf, 0xb2, 0x3b, 0x98, 0x2e, 0xf8 ),
+	EXPECTED ( 0x09, 0x3f, 0xf5, 0x17, 0x50, 0x80, 0x9d, 0x18, 0x1a, 0x9a,
+		   0x54, 0x81, 0xc5, 0x64, 0xe3, 0x7c, 0xff, 0x61, 0x8d, 0xef,
+		   0x8e, 0xc4, 0x5f, 0x46, 0x4b, 0x1a, 0x6e, 0x24, 0xf8, 0xb8,
+		   0x26, 0xbd ) );
+
+/* 0x02efc89849b3aa7efe4458a885ab9099a435a240ae5af305535ec42e0829a3b2e9 ^ -1 =
+ * 0x591607b163e89d0ac33a62c881e984a25d3826e3db5ce229af240dc58e5b579a (mod p)
+ */
+X25519_INVERT_TEST ( invert_2,
+	INVERTEND ( 0x02, 0xef, 0xc8, 0x98, 0x49, 0xb3, 0xaa, 0x7e, 0xfe,
+		    0x44, 0x58, 0xa8, 0x85, 0xab, 0x90, 0x99, 0xa4, 0x35,
+		    0xa2, 0x40, 0xae, 0x5a, 0xf3, 0x05, 0x53, 0x5e, 0xc4,
+		    0x2e, 0x08, 0x29, 0xa3, 0xb2, 0xe9 ),
+	EXPECTED ( 0x59, 0x16, 0x07, 0xb1, 0x63, 0xe8, 0x9d, 0x0a, 0xc3, 0x3a,
+		   0x62, 0xc8, 0x81, 0xe9, 0x84, 0xa2, 0x5d, 0x38, 0x26, 0xe3,
+		   0xdb, 0x5c, 0xe2, 0x29, 0xaf, 0x24, 0x0d, 0xc5, 0x8e, 0x5b,
+		   0x57, 0x9a ) );
+
+/* 0x003eabedcbbaa80dd488bd64072bcfbe01a28defe39bf0027312476f57a5e5a5ab ^ -1 =
+ * 0x7d87c2e565b27c5038181a0a7cae9ebe826c8afc1f77128a4d62cce96d2759a2 (mod p)
+ */
+X25519_INVERT_TEST ( invert_3,
+	INVERTEND ( 0x00, 0x3e, 0xab, 0xed, 0xcb, 0xba, 0xa8, 0x0d, 0xd4,
+		    0x88, 0xbd, 0x64, 0x07, 0x2b, 0xcf, 0xbe, 0x01, 0xa2,
+		    0x8d, 0xef, 0xe3, 0x9b, 0xf0, 0x02, 0x73, 0x12, 0x47,
+		    0x6f, 0x57, 0xa5, 0xe5, 0xa5, 0xab ),
+	EXPECTED ( 0x7d, 0x87, 0xc2, 0xe5, 0x65, 0xb2, 0x7c, 0x50, 0x38, 0x18,
+		   0x1a, 0x0a, 0x7c, 0xae, 0x9e, 0xbe, 0x82, 0x6c, 0x8a, 0xfc,
+		   0x1f, 0x77, 0x12, 0x8a, 0x4d, 0x62, 0xcc, 0xe9, 0x6d, 0x27,
+		   0x59, 0xa2 ) );
+
+/* 0x008e944239b02b61c4a3d70628ece66fa2fd5166e6451b4cf36123fdf77656af72 ^ -1 =
+ * 0x08e96161a0eee1b29af396f154950d5c715dc61aff66ee97377ab22adf3321d7 (mod p)
+ */
+X25519_INVERT_TEST ( invert_4,
+	INVERTEND ( 0x00, 0x8e, 0x94, 0x42, 0x39, 0xb0, 0x2b, 0x61, 0xc4,
+		    0xa3, 0xd7, 0x06, 0x28, 0xec, 0xe6, 0x6f, 0xa2, 0xfd,
+		    0x51, 0x66, 0xe6, 0x45, 0x1b, 0x4c, 0xf3, 0x61, 0x23,
+		    0xfd, 0xf7, 0x76, 0x56, 0xaf, 0x72 ),
+	EXPECTED ( 0x08, 0xe9, 0x61, 0x61, 0xa0, 0xee, 0xe1, 0xb2, 0x9a, 0xf3,
+		   0x96, 0xf1, 0x54, 0x95, 0x0d, 0x5c, 0x71, 0x5d, 0xc6, 0x1a,
+		   0xff, 0x66, 0xee, 0x97, 0x37, 0x7a, 0xb2, 0x2a, 0xdf, 0x33,
+		   0x21, 0xd7 ) );
+
+/* 0x00d261a7ab3aa2e4f90e51f30dc6a7ee39c4b032ccd7c524a55304317faf42e12f ^ -1 =
+ * 0x0738781c0aeabfbe6e840c85bd30996ef71bc54988ce16cedd5ab4f30c281597 (mod p)
+ */
+X25519_INVERT_TEST ( invert_5,
+	INVERTEND ( 0x00, 0xd2, 0x61, 0xa7, 0xab, 0x3a, 0xa2, 0xe4, 0xf9,
+		    0x0e, 0x51, 0xf3, 0x0d, 0xc6, 0xa7, 0xee, 0x39, 0xc4,
+		    0xb0, 0x32, 0xcc, 0xd7, 0xc5, 0x24, 0xa5, 0x53, 0x04,
+		    0x31, 0x7f, 0xaf, 0x42, 0xe1, 0x2f ),
+	EXPECTED ( 0x07, 0x38, 0x78, 0x1c, 0x0a, 0xea, 0xbf, 0xbe, 0x6e, 0x84,
+		   0x0c, 0x85, 0xbd, 0x30, 0x99, 0x6e, 0xf7, 0x1b, 0xc5, 0x49,
+		   0x88, 0xce, 0x16, 0xce, 0xdd, 0x5a, 0xb4, 0xf3, 0x0c, 0x28,
+		   0x15, 0x97 ) );
+
+/* Base:   0xe6db6867583030db3594c1a424b15f7c726624ec26b3353b10a903a6d0ab1c4c
+ * Scalar: 0xa546e36bf0527c9d3b16154b82465edd62144c0ac1fc5a18506a2244ba449ac4
+ * Result: 0xc3da55379de9c6908e94ea4df28d084f32eccf03491c71f754b4075577a28552
+ */
+X25519_KEY_TEST ( rfc7748_1, 1, 0,
+	BASE ( 0xe6, 0xdb, 0x68, 0x67, 0x58, 0x30, 0x30, 0xdb, 0x35, 0x94,
+	       0xc1, 0xa4, 0x24, 0xb1, 0x5f, 0x7c, 0x72, 0x66, 0x24, 0xec,
+	       0x26, 0xb3, 0x35, 0x3b, 0x10, 0xa9, 0x03, 0xa6, 0xd0, 0xab,
+	       0x1c, 0x4c ),
+	SCALAR ( 0xa5, 0x46, 0xe3, 0x6b, 0xf0, 0x52, 0x7c, 0x9d, 0x3b, 0x16,
+		 0x15, 0x4b, 0x82, 0x46, 0x5e, 0xdd, 0x62, 0x14, 0x4c, 0x0a,
+		 0xc1, 0xfc, 0x5a, 0x18, 0x50, 0x6a, 0x22, 0x44, 0xba, 0x44,
+		 0x9a, 0xc4 ),
+	EXPECTED ( 0xc3, 0xda, 0x55, 0x37, 0x9d, 0xe9, 0xc6, 0x90, 0x8e, 0x94,
+		   0xea, 0x4d, 0xf2, 0x8d, 0x08, 0x4f, 0x32, 0xec, 0xcf, 0x03,
+		   0x49, 0x1c, 0x71, 0xf7, 0x54, 0xb4, 0x07, 0x55, 0x77, 0xa2,
+		   0x85, 0x52 ) );
+
+/* Base:   0xe5210f12786811d3f4b7959d0538ae2c31dbe7106fc03c3efc4cd549c715a493
+ * Scalar: 0x4b66e9d4d1b4673c5ad22691957d6af5c11b6421e0ea01d42ca4169e7918ba0d
+ * Result: 0x95cbde9476e8907d7aade45cb4b873f88b595a68799fa152e6f8f7647aac7957
+ */
+X25519_KEY_TEST ( rfc7748_2, 1, 0,
+	BASE ( 0xe5, 0x21, 0x0f, 0x12, 0x78, 0x68, 0x11, 0xd3, 0xf4, 0xb7,
+	       0x95, 0x9d, 0x05, 0x38, 0xae, 0x2c, 0x31, 0xdb, 0xe7, 0x10,
+	       0x6f, 0xc0, 0x3c, 0x3e, 0xfc, 0x4c, 0xd5, 0x49, 0xc7, 0x15,
+	       0xa4, 0x93 ),
+	SCALAR ( 0x4b, 0x66, 0xe9, 0xd4, 0xd1, 0xb4, 0x67, 0x3c, 0x5a, 0xd2,
+		 0x26, 0x91, 0x95, 0x7d, 0x6a, 0xf5, 0xc1, 0x1b, 0x64, 0x21,
+		 0xe0, 0xea, 0x01, 0xd4, 0x2c, 0xa4, 0x16, 0x9e, 0x79, 0x18,
+		 0xba, 0x0d ),
+	EXPECTED ( 0x95, 0xcb, 0xde, 0x94, 0x76, 0xe8, 0x90, 0x7d, 0x7a, 0xad,
+		   0xe4, 0x5c, 0xb4, 0xb8, 0x73, 0xf8, 0x8b, 0x59, 0x5a, 0x68,
+		   0x79, 0x9f, 0xa1, 0x52, 0xe6, 0xf8, 0xf7, 0x64, 0x7a, 0xac,
+		   0x79, 0x57 ) );
+
+/* Base:   0x0900000000000000000000000000000000000000000000000000000000000000
+ * Scalar: 0x0900000000000000000000000000000000000000000000000000000000000000
+ * Result: 0x422c8e7a6227d7bca1350b3e2bb7279f7897b87bb6854b783c60e80311ae3079
+ */
+X25519_KEY_TEST ( rfc7748_3, 1, 0,
+	BASE ( 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	       0x00, 0x00 ),
+	SCALAR ( 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		 0x00, 0x00 ),
+	EXPECTED ( 0x42, 0x2c, 0x8e, 0x7a, 0x62, 0x27, 0xd7, 0xbc, 0xa1, 0x35,
+		   0x0b, 0x3e, 0x2b, 0xb7, 0x27, 0x9f, 0x78, 0x97, 0xb8, 0x7b,
+		   0xb6, 0x85, 0x4b, 0x78, 0x3c, 0x60, 0xe8, 0x03, 0x11, 0xae,
+		   0x30, 0x79 ) );
+
+/* Base:   0x0900000000000000000000000000000000000000000000000000000000000000
+ * Scalar: 0x0900000000000000000000000000000000000000000000000000000000000000
+ * Result: 0xb1a5a73158904c020866c13939dd7e1aa26852ee1d2609c92e5a8f1debe2150a
+ * (after 100 iterations)
+ *
+ * RFC7748 gives test vectors for 1000 and 1000000 iterations with
+ * these starting values.  This test case stops after 100 iterations
+ * to avoid a pointlessly slow test cycle in the common case of
+ * running tests under Valgrind.
+ */
+X25519_KEY_TEST ( rfc7748_4_100, 100, 0,
+	BASE ( 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	       0x00, 0x00 ),
+	SCALAR ( 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		 0x00, 0x00 ),
+	EXPECTED ( 0xb1, 0xa5, 0xa7, 0x31, 0x58, 0x90, 0x4c, 0x02, 0x08, 0x66,
+		   0xc1, 0x39, 0x39, 0xdd, 0x7e, 0x1a, 0xa2, 0x68, 0x52, 0xee,
+		   0x1d, 0x26, 0x09, 0xc9, 0x2e, 0x5a, 0x8f, 0x1d, 0xeb, 0xe2,
+		   0x15, 0x0a ) );
+
+/* Base:   2^255 - 19 + 1 (deliberately malicious public key)
+ * Scalar: 0x000102030405060708090a0b0c0d0e0f000102030405060708090a0b0c0d0e0f
+ * Result: Failure (all zeros)
+ */
+X25519_KEY_TEST ( malicious, 1, 1,
+	BASE ( 0xee, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	       0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	       0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	       0xff, 0x7f ),
+	SCALAR ( 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09,
+		 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x00, 0x01, 0x02, 0x03,
+		 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d,
+		 0x0e, 0x0f ),
+	EXPECTED ( 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		   0x00, 0x00 ) );
+
+/**
+ * Perform X25519 self-tests
+ *
+ */
+static void x25519_test_exec ( void ) {
+
+	/* Perform multiplication tests */
+	x25519_multiply_ok ( &multiply_small );
+	x25519_multiply_ok ( &multiply_k_p );
+	x25519_multiply_ok ( &multiply_1 );
+	x25519_multiply_ok ( &multiply_2 );
+	x25519_multiply_ok ( &multiply_3 );
+	x25519_multiply_ok ( &multiply_4 );
+	x25519_multiply_ok ( &multiply_5 );
+
+	/* Perform multiplicative inversion tests */
+	x25519_invert_ok ( &invert_1 );
+	x25519_invert_ok ( &invert_2 );
+	x25519_invert_ok ( &invert_3 );
+	x25519_invert_ok ( &invert_4 );
+	x25519_invert_ok ( &invert_5 );
+
+	/* Perform key exchange tests */
+	x25519_key_ok ( &rfc7748_1 );
+	x25519_key_ok ( &rfc7748_2 );
+	x25519_key_ok ( &rfc7748_3 );
+	x25519_key_ok ( &rfc7748_4_100 );
+	x25519_key_ok ( &malicious );
+}
+
+/** X25519 self-test */
+struct self_test x25519_test __self_test = {
+	.name = "x25519",
+	.exec = x25519_test_exec,
+};
diff --git a/src/tests/x509_test.c b/src/tests/x509_test.c
index b6cba575..50eb4d78 100644
--- a/src/tests/x509_test.c
+++ b/src/tests/x509_test.c
@@ -984,6 +984,7 @@ static void x509_validate_chain_fail_okx ( struct x509_test_chain *chn,
  *
  */
 static void x509_test_exec ( void ) {
+	struct x509_link *link;
 
 	/* Parse all certificates */
 	x509_certificate_ok ( &root_crt );
@@ -1089,6 +1090,22 @@ static void x509_test_exec ( void ) {
 	x509_validate_chain_fail_ok ( &useless_chain, test_ca_expired,
 				      &empty_store, &test_root );
 
+	/* Check chain truncation */
+	link = list_last_entry ( &server_chain.chain->links,
+				 struct x509_link, list );
+	ok ( link->cert == root_crt.cert );
+	link = list_prev_entry ( link, &server_chain.chain->links, list );
+	ok ( link->cert == intermediate_crt.cert );
+	x509_validate_chain_ok ( &server_chain, test_time,
+				 &empty_store, &test_root );
+	x509_truncate ( server_chain.chain, link );
+	x509_validate_chain_fail_ok ( &server_chain, test_time,
+				      &empty_store, &test_root );
+
+	/* Check self-signedess */
+	ok ( x509_is_self_signed ( root_crt.cert ) );
+	ok ( ! x509_is_self_signed ( intermediate_crt.cert ) );
+
 	/* Sanity check */
 	assert ( list_empty ( &empty_store.links ) );
 
diff --git a/src/usr/autoboot.c b/src/usr/autoboot.c
index d1f25962..4b64ca82 100644
--- a/src/usr/autoboot.c
+++ b/src/usr/autoboot.c
@@ -116,7 +116,7 @@ const struct setting skip_san_boot_setting __setting ( SETTING_SANBOOT_EXTRA,
  * @v root_paths	Root path(s)
  * @v root_path_count	Number of root paths
  * @v drive		SAN drive (if applicable)
- * @v san_filename	SAN filename (or NULL to use default)
+ * @v san_config	SAN boot configuration parameters
  * @v flags		Boot action flags
  * @ret rc		Return status code
  *
@@ -128,8 +128,9 @@ const struct setting skip_san_boot_setting __setting ( SETTING_SANBOOT_EXTRA,
  */
 int uriboot ( struct uri *filename, struct uri **root_paths,
 	      unsigned int root_path_count, int drive,
-	      const char *san_filename, unsigned int flags ) {
+	      struct san_boot_config *san_config, unsigned int flags ) {
 	struct image *image;
+	const char *san_filename;
 	int rc;
 
 	/* Hook SAN device, if applicable */
@@ -182,11 +183,12 @@ int uriboot ( struct uri *filename, struct uri **root_paths,
 
 	/* Attempt SAN boot if applicable */
 	if ( ! ( flags & URIBOOT_NO_SAN_BOOT ) ) {
+		san_filename = san_config->filename;
 		if ( fetch_intz_setting ( NULL, &skip_san_boot_setting) == 0 ) {
 			printf ( "Booting%s%s from SAN device %#02x\n",
 				 ( san_filename ? " " : "" ),
 				 ( san_filename ? san_filename : "" ), drive );
-			rc = san_boot ( drive, san_filename );
+			rc = san_boot ( drive, san_config );
 			printf ( "Boot from SAN device %#02x failed: %s\n",
 				 drive, strerror ( rc ) );
 		} else {
@@ -387,6 +389,7 @@ static int have_pxe_menu ( void ) {
  * @ret rc		Return status code
  */
 int netboot ( struct net_device *netdev ) {
+	struct san_boot_config san_config;
 	struct uri *filename;
 	struct uri *root_path;
 	char *san_filename;
@@ -421,6 +424,10 @@ int netboot ( struct net_device *netdev ) {
 	/* Fetch SAN filename (if any) */
 	san_filename = fetch_san_filename ( NULL );
 
+	/* Construct SAN boot configuration parameters */
+	memset ( &san_config, 0, sizeof ( san_config ) );
+	san_config.filename = san_filename;
+
 	/* If we have both a filename and a root path, ignore an
 	 * unsupported or missing URI scheme in the root path, since
 	 * it may represent an NFS root.
@@ -442,7 +449,7 @@ int netboot ( struct net_device *netdev ) {
 
 	/* Boot using next server, filename and root path */
 	if ( ( rc = uriboot ( filename, &root_path, ( root_path ? 1 : 0 ),
-			      san_default_drive(), san_filename,
+			      san_default_drive(), &san_config,
 			      ( root_path ? 0 : URIBOOT_NO_SAN ) ) ) != 0 )
 		goto err_uriboot;
 
diff --git a/src/util/elf2efi.c b/src/util/elf2efi.c
index 5b3e785f..4af587d8 100644
--- a/src/util/elf2efi.c
+++ b/src/util/elf2efi.c
@@ -50,6 +50,7 @@
 #define EFI_IMAGE_FILE_MACHINE		EFI_IMAGE_FILE_32BIT_MACHINE
 #define ELFCLASS   ELFCLASS32
 #define Elf_Ehdr   Elf32_Ehdr
+#define Elf_Phdr   Elf32_Phdr
 #define Elf_Shdr   Elf32_Shdr
 #define Elf_Sym    Elf32_Sym
 #define Elf_Addr   Elf32_Addr
@@ -65,6 +66,7 @@
 #define EFI_IMAGE_FILE_MACHINE		0
 #define ELFCLASS   ELFCLASS64
 #define Elf_Ehdr   Elf64_Ehdr
+#define Elf_Phdr   Elf64_Phdr
 #define Elf_Shdr   Elf64_Shdr
 #define Elf_Sym    Elf64_Sym
 #define Elf_Addr   Elf64_Addr
@@ -120,6 +122,9 @@
 #ifndef R_AARCH64_LDST64_ABS_LO12_NC
 #define R_AARCH64_LDST64_ABS_LO12_NC 286
 #endif
+#ifndef R_AARCH64_LDST128_ABS_LO12_NC
+#define R_AARCH64_LDST128_ABS_LO12_NC 299
+#endif
 #ifndef R_ARM_CALL
 #define R_ARM_CALL 28
 #endif
@@ -135,6 +140,12 @@
 #ifndef R_LARCH_64
 #define R_LARCH_64 2
 #endif
+#ifndef R_LARCH_B16
+#define R_LARCH_B16 64
+#endif
+#ifndef R_LARCH_B21
+#define R_LARCH_B21 65
+#endif
 #ifndef R_LARCH_B26
 #define R_LARCH_B26 66
 #endif
@@ -150,6 +161,18 @@
 #ifndef R_LARCH_GOT_PC_LO12
 #define R_LARCH_GOT_PC_LO12 76
 #endif
+#ifndef R_LARCH_RELAX
+#define R_LARCH_RELAX 100
+#endif
+#ifndef R_LARCH_PCREL20_S2
+#define R_LARCH_PCREL20_S2 103
+#endif
+#ifndef R_X86_64_GOTPCRELX
+#define R_X86_64_GOTPCRELX 41
+#endif
+#ifndef R_X86_64_REX_GOTPCRELX
+#define R_X86_64_REX_GOTPCRELX 42
+#endif
 
 /**
  * Alignment of raw data of sections in the image file
@@ -168,6 +191,9 @@
  */
 #define EFI_IMAGE_ALIGN 0x1000
 
+/** Set PointerToRawData automatically */
+#define PTRD_AUTO 0xffffffffUL
+
 /** Number of data directory entries */
 #define NUMBER_OF_DIRECTORY_ENTRIES 8
 
@@ -180,7 +206,6 @@ struct elf_file {
 struct pe_section {
 	struct pe_section *next;
 	EFI_IMAGE_SECTION_HEADER hdr;
-	void ( * fixup ) ( struct pe_section *section );
 	int hidden;
 	uint8_t contents[0];
 };
@@ -215,6 +240,7 @@ static struct pe_header efi_pe_header = {
 					     DataDirectory[0] ) ) ),
 			.Characteristics = ( EFI_IMAGE_FILE_DLL |
 					     EFI_IMAGE_FILE_MACHINE |
+					     EFI_IMAGE_FILE_LARGE_ADDRESS_AWARE|
 					     EFI_IMAGE_FILE_EXECUTABLE_IMAGE ),
 		},
 		.OptionalHeader = {
@@ -224,7 +250,14 @@ static struct pe_header efi_pe_header = {
 			.SectionAlignment = EFI_IMAGE_ALIGN,
 			.FileAlignment = EFI_FILE_ALIGN,
 			.SizeOfImage = EFI_IMAGE_ALIGN,
-			.SizeOfHeaders = sizeof ( efi_pe_header ),
+			.SizeOfHeaders =
+				( sizeof ( efi_pe_header ) -
+				  ( ( EFI_IMAGE_NUMBER_OF_DIRECTORY_ENTRIES -
+				      NUMBER_OF_DIRECTORY_ENTRIES ) *
+				    sizeof ( efi_pe_header.nt.OptionalHeader.
+					     DataDirectory[0] ) ) ),
+			.DllCharacteristics =
+				IMAGE_DLLCHARACTERISTICS_NX_COMPAT,
 			.NumberOfRvaAndSizes = NUMBER_OF_DIRECTORY_ENTRIES,
 		},
 	},
@@ -426,6 +459,14 @@ static void read_elf_file ( const char *name, struct elf_file *elf ) {
 	}
 	elf->ehdr = ehdr;
 
+	/* Check program headers */
+	if ( ( elf->len < ehdr->e_phoff ) ||
+	     ( ( elf->len - ehdr->e_phoff ) <
+	       ( ( ( unsigned int ) ehdr->e_phnum ) * ehdr->e_phentsize ) ) ) {
+		eprintf ( "ELF program headers outside file in %s\n", name );
+		exit ( 1 );
+	}
+
 	/* Check section headers */
 	for ( i = 0 ; i < ehdr->e_shnum ; i++ ) {
 		offset = ( ehdr->e_shoff + ( i * ehdr->e_shentsize ) );
@@ -496,6 +537,39 @@ static const char * elf_string ( struct elf_file *elf, unsigned int section,
 }
 
 /**
+ * Get section load memory address
+ *
+ * @v elf		ELF file
+ * @v shdr		ELF section header
+ * @v name		ELF section name
+ * @ret lma		Load memory address
+ */
+static unsigned long elf_lma ( struct elf_file *elf, const Elf_Shdr *shdr,
+			       const char *name ) {
+	const Elf_Ehdr *ehdr = elf->ehdr;
+	const Elf_Phdr *phdr;
+	size_t offset;
+	unsigned int i;
+
+	/* Find containing segment */
+	for ( i = 0 ; i < ehdr->e_phnum ; i++ ) {
+		offset = ( ehdr->e_phoff + ( i * ehdr->e_phentsize ) );
+		phdr = ( elf->data + offset );
+		if ( ( phdr->p_type == PT_LOAD ) &&
+		     ( phdr->p_vaddr <= shdr->sh_addr ) &&
+		     ( ( shdr->sh_addr - phdr->p_vaddr + shdr->sh_size ) <=
+		       phdr->p_memsz ) ) {
+			/* Found matching segment */
+			return ( phdr->p_paddr +
+				 ( shdr->sh_addr - phdr->p_vaddr ) );
+		}
+	}
+
+	eprintf ( "No containing segment for section %s\n", name );
+	exit ( 1 );
+}
+
+/**
  * Set machine architecture
  *
  * @v elf		ELF file
@@ -537,41 +611,41 @@ static void set_machine ( struct elf_file *elf, struct pe_header *pe_header ) {
  * @v elf		ELF file
  * @v shdr		ELF section header
  * @v pe_header		PE file header
+ * @v opts		Options
  * @ret new		New PE section
  */
 static struct pe_section * process_section ( struct elf_file *elf,
 					     const Elf_Shdr *shdr,
-					     struct pe_header *pe_header ) {
+					     struct pe_header *pe_header,
+					     struct options *opts ) {
 	struct pe_section *new;
 	const char *name;
 	size_t name_len;
 	size_t section_memsz;
 	size_t section_filesz;
-	unsigned long code_start;
-	unsigned long code_end;
-	unsigned long data_start;
-	unsigned long data_mid;
-	unsigned long data_end;
-	unsigned long start;
-	unsigned long end;
-	unsigned long *applicable_start;
-	unsigned long *applicable_end;
+	uint32_t start;
+	uint32_t end;
+	uint32_t *code_start;
+	uint32_t *data_start;
+	uint32_t *code_size;
+	uint32_t *data_size;
+	uint32_t *bss_size;
+	uint32_t *applicable_start;
+	uint32_t *applicable_size;
 
 	/* Get section name */
 	name = elf_string ( elf, elf->ehdr->e_shstrndx, shdr->sh_name );
 
-	/* Extract current RVA limits from file header */
-	code_start = pe_header->nt.OptionalHeader.BaseOfCode;
-	code_end = ( code_start + pe_header->nt.OptionalHeader.SizeOfCode );
+	/* Identify start and size limit fields from file header */
+	code_start = &pe_header->nt.OptionalHeader.BaseOfCode;
+	code_size = &pe_header->nt.OptionalHeader.SizeOfCode;
 #if defined(EFI_TARGET32)
-	data_start = pe_header->nt.OptionalHeader.BaseOfData;
+	data_start = &pe_header->nt.OptionalHeader.BaseOfData;
 #elif defined(EFI_TARGET64)
-	data_start = code_end;
+	data_start = NULL;
 #endif
-	data_mid = ( data_start +
-		     pe_header->nt.OptionalHeader.SizeOfInitializedData );
-	data_end = ( data_mid +
-		     pe_header->nt.OptionalHeader.SizeOfUninitializedData );
+	data_size = &pe_header->nt.OptionalHeader.SizeOfInitializedData;
+	bss_size = &pe_header->nt.OptionalHeader.SizeOfUninitializedData;
 
 	/* Allocate PE section */
 	section_memsz = shdr->sh_size;
@@ -588,36 +662,51 @@ static struct pe_section * process_section ( struct elf_file *elf,
 	new->hdr.Misc.VirtualSize = section_memsz;
 	new->hdr.VirtualAddress = shdr->sh_addr;
 	new->hdr.SizeOfRawData = section_filesz;
+	if ( shdr->sh_type == SHT_PROGBITS ) {
+		if ( opts->hybrid ) {
+			new->hdr.PointerToRawData = elf_lma ( elf, shdr, name );
+			if ( new->hdr.PointerToRawData == 0 )
+				new->hidden = 1;
+		} else {
+			new->hdr.PointerToRawData = PTRD_AUTO;
+		}
+	}
 
-	/* Fill in section characteristics and update RVA limits */
+	/* Treat 16-bit sections as hidden in hybrid binaries */
+	if ( opts->hybrid && ( strlen ( name ) > 2 ) &&
+	     ( strcmp ( &name[ strlen ( name ) - 2 ], "16" ) == 0 ) ) {
+		new->hidden = 1;
+	}
+
+	/* Fill in section characteristics and identify applicable limits */
 	if ( ( shdr->sh_type == SHT_PROGBITS ) &&
-	     ( shdr->sh_flags & SHF_EXECINSTR ) ) {
-		/* .text-type section */
-		new->hdr.Characteristics =
-			( EFI_IMAGE_SCN_CNT_CODE |
-			  EFI_IMAGE_SCN_MEM_NOT_PAGED |
-			  EFI_IMAGE_SCN_MEM_EXECUTE |
-			  EFI_IMAGE_SCN_MEM_READ );
-		applicable_start = &code_start;
-		applicable_end = &code_end;
-	} else if ( ( shdr->sh_type == SHT_PROGBITS ) &&
-		    ( shdr->sh_flags & SHF_WRITE ) ) {
+	     ( shdr->sh_flags & SHF_WRITE ) ) {
 		/* .data-type section */
 		new->hdr.Characteristics =
 			( EFI_IMAGE_SCN_CNT_INITIALIZED_DATA |
 			  EFI_IMAGE_SCN_MEM_NOT_PAGED |
 			  EFI_IMAGE_SCN_MEM_READ |
 			  EFI_IMAGE_SCN_MEM_WRITE );
-		applicable_start = &data_start;
-		applicable_end = &data_mid;
+		applicable_start = data_start;
+		applicable_size = data_size;
+	} else if ( ( shdr->sh_type == SHT_PROGBITS ) &&
+		    ( shdr->sh_flags & SHF_EXECINSTR ) ) {
+		/* .text-type section */
+		new->hdr.Characteristics =
+			( EFI_IMAGE_SCN_CNT_CODE |
+			  EFI_IMAGE_SCN_MEM_NOT_PAGED |
+			  EFI_IMAGE_SCN_MEM_EXECUTE |
+			  EFI_IMAGE_SCN_MEM_READ );
+		applicable_start = code_start;
+		applicable_size = code_size;
 	} else if ( shdr->sh_type == SHT_PROGBITS ) {
 		/* .rodata-type section */
 		new->hdr.Characteristics =
 			( EFI_IMAGE_SCN_CNT_INITIALIZED_DATA |
 			  EFI_IMAGE_SCN_MEM_NOT_PAGED |
 			  EFI_IMAGE_SCN_MEM_READ );
-		applicable_start = &data_start;
-		applicable_end = &data_mid;
+		applicable_start = data_start;
+		applicable_size = data_size;
 	} else if ( shdr->sh_type == SHT_NOBITS ) {
 		/* .bss-type section */
 		new->hdr.Characteristics =
@@ -625,8 +714,8 @@ static struct pe_section * process_section ( struct elf_file *elf,
 			  EFI_IMAGE_SCN_MEM_NOT_PAGED |
 			  EFI_IMAGE_SCN_MEM_READ |
 			  EFI_IMAGE_SCN_MEM_WRITE );
-		applicable_start = &data_mid;
-		applicable_end = &data_end;
+		applicable_start = data_start;
+		applicable_size = bss_size;
 	} else {
 		eprintf ( "Unrecognised characteristics for section %s\n",
 			  name );
@@ -639,46 +728,70 @@ static struct pe_section * process_section ( struct elf_file *elf,
 			 shdr->sh_size );
 	}
 
-	/* Update RVA limits */
+	/* Update file header fields */
 	start = new->hdr.VirtualAddress;
-	end = ( start + new->hdr.Misc.VirtualSize );
-	if ( ! new->hidden ) {
-		if ( ( ! *applicable_start ) || ( *applicable_start >= start ) )
-			*applicable_start = start;
-		if ( *applicable_end < end )
-			*applicable_end = end;
-	}
-	if ( data_start < code_end )
-		data_start = code_end;
-	if ( data_mid < data_start )
-		data_mid = data_start;
-	if ( data_end < data_mid )
-		data_end = data_mid;
-
-	/* Write RVA limits back to file header */
-	pe_header->nt.OptionalHeader.BaseOfCode = code_start;
-	pe_header->nt.OptionalHeader.SizeOfCode = ( code_end - code_start );
-#if defined(EFI_TARGET32)
-	pe_header->nt.OptionalHeader.BaseOfData = data_start;
-#endif
-	pe_header->nt.OptionalHeader.SizeOfInitializedData =
-		( data_mid - data_start );
-	pe_header->nt.OptionalHeader.SizeOfUninitializedData =
-		( data_end - data_mid );
-
-	/* Update remaining file header fields */
 	if ( ! new->hidden ) {
 		pe_header->nt.FileHeader.NumberOfSections++;
 		pe_header->nt.OptionalHeader.SizeOfHeaders +=
 			sizeof ( new->hdr );
+		if ( applicable_start && ( ( *applicable_start == 0 ) ||
+					   ( start < *applicable_start ) ) ) {
+			*applicable_start = start;
+		}
+		if ( applicable_size ) {
+			*applicable_size += section_memsz;
+		}
+	}
+	end = efi_image_align ( start + section_memsz );
+	if ( end > pe_header->nt.OptionalHeader.SizeOfImage ) {
+		pe_header->nt.OptionalHeader.SizeOfImage = end;
 	}
-	pe_header->nt.OptionalHeader.SizeOfImage =
-		efi_image_align ( data_end );
 
 	return new;
 }
 
 /**
+ * Update image base address
+ *
+ * @v pe_header		PE file header
+ * @v pe_sections	List of PE sections
+ * @v pe_reltab		PE relocation table
+ */
+static void update_image_base ( struct pe_header *pe_header,
+				struct pe_section *pe_sections,
+				struct pe_relocs *pe_reltab ) {
+	struct pe_section *section;
+	struct pe_relocs *pe_rel;
+	unsigned long base = -1UL;
+
+	/* Set ImageBase to the highest possible value, leaving space
+	 * for the PE header itself.
+	 */
+	for ( section = pe_sections ; section ; section = section->next ) {
+		if ( ! section->hidden ) {
+			if ( base > section->hdr.VirtualAddress )
+				base = section->hdr.VirtualAddress;
+		}
+	}
+	base -= EFI_IMAGE_ALIGN;
+	pe_header->nt.OptionalHeader.ImageBase = base;
+
+	/* Adjust RVAs to match ImageBase */
+	pe_header->nt.OptionalHeader.AddressOfEntryPoint -= base;
+	pe_header->nt.OptionalHeader.BaseOfCode -= base;
+#if defined(EFI_TARGET32)
+	pe_header->nt.OptionalHeader.BaseOfData -= base;
+#endif
+	pe_header->nt.OptionalHeader.SizeOfImage -= base;
+	for ( section = pe_sections ; section ; section = section->next ) {
+		section->hdr.VirtualAddress -= base;
+	}
+	for ( pe_rel = pe_reltab ; pe_rel ; pe_rel = pe_rel->next ) {
+		pe_rel->start_rva -= base;
+	}
+}
+
+/**
  * Process relocation record
  *
  * @v elf		ELF file
@@ -736,6 +849,8 @@ static void process_reloc ( struct elf_file *elf, const Elf_Shdr *shdr,
 		case ELF_MREL ( EM_ARM, R_ARM_V4BX ):
 		case ELF_MREL ( EM_X86_64, R_X86_64_PC32 ) :
 		case ELF_MREL ( EM_X86_64, R_X86_64_PLT32 ) :
+		case ELF_MREL ( EM_X86_64, R_X86_64_GOTPCRELX ) :
+		case ELF_MREL ( EM_X86_64, R_X86_64_REX_GOTPCRELX ) :
 		case ELF_MREL ( EM_AARCH64, R_AARCH64_CALL26 ) :
 		case ELF_MREL ( EM_AARCH64, R_AARCH64_JUMP26 ) :
 		case ELF_MREL ( EM_AARCH64, R_AARCH64_ADR_PREL_LO21 ) :
@@ -745,16 +860,25 @@ static void process_reloc ( struct elf_file *elf, const Elf_Shdr *shdr,
 		case ELF_MREL ( EM_AARCH64, R_AARCH64_LDST16_ABS_LO12_NC ) :
 		case ELF_MREL ( EM_AARCH64, R_AARCH64_LDST32_ABS_LO12_NC ) :
 		case ELF_MREL ( EM_AARCH64, R_AARCH64_LDST64_ABS_LO12_NC ) :
+		case ELF_MREL ( EM_AARCH64, R_AARCH64_LDST128_ABS_LO12_NC ) :
+		case ELF_MREL ( EM_LOONGARCH, R_LARCH_B16):
+		case ELF_MREL ( EM_LOONGARCH, R_LARCH_B21):
 		case ELF_MREL ( EM_LOONGARCH, R_LARCH_B26):
 		case ELF_MREL ( EM_LOONGARCH, R_LARCH_PCALA_HI20 ):
 		case ELF_MREL ( EM_LOONGARCH, R_LARCH_PCALA_LO12 ):
 		case ELF_MREL ( EM_LOONGARCH, R_LARCH_GOT_PC_HI20 ):
 		case ELF_MREL ( EM_LOONGARCH, R_LARCH_GOT_PC_LO12 ):
+		case ELF_MREL ( EM_LOONGARCH, R_LARCH_PCREL20_S2 ):
 			/* Skip PC-relative relocations; all relative
 			 * offsets remain unaltered when the object is
 			 * loaded.
 			 */
 			break;
+		case ELF_MREL ( EM_LOONGARCH, R_LARCH_RELAX ):
+			/* Relocation can be relaxed (optimized out).
+			 * Ignore it for now.
+			 */
+			break;
 		case ELF_MREL ( EM_X86_64, R_X86_64_32 ) :
 			/* Ignore 32-bit relocations in a hybrid
 			 * 32-bit BIOS and 64-bit UEFI binary,
@@ -832,6 +956,7 @@ create_reloc_section ( struct pe_header *pe_header,
 	reloc->hdr.Misc.VirtualSize = section_memsz;
 	reloc->hdr.VirtualAddress = pe_header->nt.OptionalHeader.SizeOfImage;
 	reloc->hdr.SizeOfRawData = section_filesz;
+	reloc->hdr.PointerToRawData = PTRD_AUTO;
 	reloc->hdr.Characteristics = ( EFI_IMAGE_SCN_CNT_INITIALIZED_DATA |
 				       EFI_IMAGE_SCN_MEM_DISCARDABLE |
 				       EFI_IMAGE_SCN_MEM_NOT_PAGED |
@@ -854,20 +979,6 @@ create_reloc_section ( struct pe_header *pe_header,
 }
 
 /**
- * Fix up debug section
- *
- * @v debug		Debug section
- */
-static void fixup_debug_section ( struct pe_section *debug ) {
-	EFI_IMAGE_DEBUG_DIRECTORY_ENTRY *contents;
-
-	/* Fix up FileOffset */
-	contents = ( ( void * ) debug->contents );
-	contents->FileOffset += ( debug->hdr.PointerToRawData -
-				  debug->hdr.VirtualAddress );
-}
-
-/**
  * Create debug section
  *
  * @v pe_header		PE file header
@@ -882,27 +993,31 @@ create_debug_section ( struct pe_header *pe_header, const char *filename ) {
 	struct {
 		EFI_IMAGE_DEBUG_DIRECTORY_ENTRY debug;
 		EFI_IMAGE_DEBUG_CODEVIEW_RSDS_ENTRY rsds;
-		char name[ strlen ( filename ) + 1 ];
+		char name[32];
 	} *contents;
 
 	/* Allocate PE section */
-	section_memsz = sizeof ( *contents );
-	section_filesz = efi_file_align ( section_memsz );
+	section_filesz = section_memsz = sizeof ( *contents );
 	debug = xmalloc ( sizeof ( *debug ) + section_filesz );
 	memset ( debug, 0, sizeof ( *debug ) + section_filesz );
 	contents = ( void * ) debug->contents;
 
+	/* Place at end of headers */
+	pe_header->nt.OptionalHeader.SizeOfHeaders += sizeof ( *contents );
+	pe_header->nt.OptionalHeader.SizeOfHeaders =
+		efi_file_align ( pe_header->nt.OptionalHeader.SizeOfHeaders );
+	pe_header->nt.OptionalHeader.SizeOfHeaders -= sizeof ( *contents );
+
 	/* Fill in section header details */
 	strncpy ( ( char * ) debug->hdr.Name, ".debug",
 		  sizeof ( debug->hdr.Name ) );
 	debug->hdr.Misc.VirtualSize = section_memsz;
-	debug->hdr.VirtualAddress = pe_header->nt.OptionalHeader.SizeOfImage;
+	debug->hdr.VirtualAddress =
+		pe_header->nt.OptionalHeader.SizeOfHeaders;
 	debug->hdr.SizeOfRawData = section_filesz;
-	debug->hdr.Characteristics = ( EFI_IMAGE_SCN_CNT_INITIALIZED_DATA |
-				       EFI_IMAGE_SCN_MEM_DISCARDABLE |
-				       EFI_IMAGE_SCN_MEM_NOT_PAGED |
-				       EFI_IMAGE_SCN_MEM_READ );
-	debug->fixup = fixup_debug_section;
+	debug->hdr.PointerToRawData =
+		pe_header->nt.OptionalHeader.SizeOfHeaders;
+	debug->hidden = 1;
 
 	/* Create section contents */
 	contents->debug.TimeDateStamp = 0x10d1a884;
@@ -917,10 +1032,7 @@ create_debug_section ( struct pe_header *pe_header, const char *filename ) {
 		   filename );
 
 	/* Update file header details */
-	pe_header->nt.FileHeader.NumberOfSections++;
-	pe_header->nt.OptionalHeader.SizeOfHeaders += sizeof ( debug->hdr );
-	pe_header->nt.OptionalHeader.SizeOfImage +=
-		efi_image_align ( section_memsz );
+	pe_header->nt.OptionalHeader.SizeOfHeaders += sizeof ( *contents );
 	debugdir = &(pe_header->nt.OptionalHeader.DataDirectory
 		     [EFI_IMAGE_DIRECTORY_ENTRY_DEBUG]);
 	debugdir->VirtualAddress = debug->hdr.VirtualAddress;
@@ -940,25 +1052,79 @@ static void write_pe_file ( struct pe_header *pe_header,
 			    struct pe_section *pe_sections,
 			    FILE *pe ) {
 	struct pe_section *section;
-	unsigned long fpos = 0;
+	unsigned long hdrmax;
+	unsigned long fpos;
+	unsigned long fposmax;
 	unsigned int count = 0;
 
+	/* Extend header length to reach first explicitly placed section */
+	hdrmax = -1UL;
+	for ( section = pe_sections ; section ; section = section->next ) {
+		if ( ( section->hdr.PointerToRawData != PTRD_AUTO ) &&
+		     ( section->hdr.SizeOfRawData > 0 ) &&
+		     ( ! section->hidden ) &&
+		     ( hdrmax > section->hdr.PointerToRawData ) ) {
+			hdrmax = section->hdr.PointerToRawData;
+		}
+	}
+	if ( ( hdrmax != -1UL ) &&
+	     ( pe_header->nt.OptionalHeader.SizeOfHeaders < hdrmax ) ) {
+		pe_header->nt.OptionalHeader.SizeOfHeaders = hdrmax;
+	}
+
 	/* Align length of headers */
-	fpos = pe_header->nt.OptionalHeader.SizeOfHeaders =
+	fpos = fposmax = pe_header->nt.OptionalHeader.SizeOfHeaders =
 		efi_file_align ( pe_header->nt.OptionalHeader.SizeOfHeaders );
+	if ( fpos > hdrmax ) {
+		eprintf ( "Cannot fit %lx bytes of headers before section at "
+			  "file offset %lx\n", fpos, hdrmax );
+		exit ( 1 );
+	}
 
 	/* Assign raw data pointers */
 	for ( section = pe_sections ; section ; section = section->next ) {
-		if ( section->hdr.SizeOfRawData ) {
-			section->hdr.PointerToRawData = fpos;
-			fpos += section->hdr.SizeOfRawData;
-			fpos = efi_file_align ( fpos );
+		if ( section->hdr.PointerToRawData == PTRD_AUTO ) {
+			fpos = fposmax;
+		} else {
+			fpos = section->hdr.PointerToRawData;
+		}
+		section->hdr.PointerToRawData = fpos;
+		fpos += section->hdr.SizeOfRawData;
+		fpos = efi_file_align ( fpos );
+		if ( fpos > fposmax )
+			fposmax = fpos;
+	}
+
+	/* Write sections */
+	for ( section = pe_sections ; section ; section = section->next ) {
+		if ( ( section->hdr.PointerToRawData & ( EFI_FILE_ALIGN - 1 ) )
+		     && ( ! section->hidden ) ) {
+			eprintf ( "Section %.8s file offset %x is "
+				  "misaligned\n", section->hdr.Name,
+				  section->hdr.PointerToRawData );
+			exit ( 1 );
+		}
+		if ( fseek ( pe, section->hdr.PointerToRawData,
+			     SEEK_SET ) != 0 ) {
+			eprintf ( "Could not seek to %x: %s\n",
+				  section->hdr.PointerToRawData,
+				  strerror ( errno ) );
+			exit ( 1 );
+		}
+		if ( section->hdr.SizeOfRawData &&
+		     ( fwrite ( section->contents, section->hdr.SizeOfRawData,
+				1, pe ) != 1 ) ) {
+			eprintf ( "Could not write section %.8s: %s\n",
+				  section->hdr.Name, strerror ( errno ) );
+			exit ( 1 );
 		}
-		if ( section->fixup )
-			section->fixup ( section );
 	}
 
 	/* Write file header */
+	if ( fseek ( pe, 0, SEEK_SET ) != 0 ) {
+		eprintf ( "Could not rewind: %s\n", strerror ( errno ) );
+		exit ( 1 );
+	}
 	if ( fwrite ( pe_header,
 		      ( offsetof ( typeof ( *pe_header ), nt.OptionalHeader ) +
 			pe_header->nt.FileHeader.SizeOfOptionalHeader ),
@@ -979,24 +1145,6 @@ static void write_pe_file ( struct pe_header *pe_header,
 		count++;
 	}
 	assert ( count == pe_header->nt.FileHeader.NumberOfSections );
-
-	/* Write sections */
-	for ( section = pe_sections ; section ; section = section->next ) {
-		if ( fseek ( pe, section->hdr.PointerToRawData,
-			     SEEK_SET ) != 0 ) {
-			eprintf ( "Could not seek to %x: %s\n",
-				  section->hdr.PointerToRawData,
-				  strerror ( errno ) );
-			exit ( 1 );
-		}
-		if ( section->hdr.SizeOfRawData &&
-		     ( fwrite ( section->contents, section->hdr.SizeOfRawData,
-				1, pe ) != 1 ) ) {
-			eprintf ( "Could not write section %.8s: %s\n",
-				  section->hdr.Name, strerror ( errno ) );
-			exit ( 1 );
-		}
-	}
 }
 
 /**
@@ -1041,7 +1189,8 @@ static void elf2pe ( const char *elf_name, const char *pe_name,
 
 			/* Create output section */
 			*(next_pe_section) = process_section ( &elf, shdr,
-							       &pe_header );
+							       &pe_header,
+							       opts );
 			next_pe_section = &(*next_pe_section)->next;
 
 		} else if ( shdr->sh_type == SHT_REL ) {
@@ -1058,6 +1207,9 @@ static void elf2pe ( const char *elf_name, const char *pe_name,
 		}
 	}
 
+	/* Update image base address */
+	update_image_base ( &pe_header, pe_sections, pe_reltab );
+
 	/* Create the .reloc section */
 	*(next_pe_section) = create_reloc_section ( &pe_header, pe_reltab );
 	next_pe_section = &(*next_pe_section)->next;
diff --git a/src/util/genfsimg b/src/util/genfsimg
index 0c069279..a981a62d 100755
--- a/src/util/genfsimg
+++ b/src/util/genfsimg
@@ -269,6 +269,10 @@ if [ -n "${FATIMG}" ] ; then
 	FATSIZE=$(( FATCYLS * 504 ))
 	FATARGS="-s 63 -h 16 -t ${FATCYLS}"
     fi
+    if [ -n "${SOURCE_DATE_EPOCH:-}" ] ; then
+	FATSERIAL=$(( SOURCE_DATE_EPOCH % 100000000 ))
+	FATARGS="${FATARGS} -N ${FATSERIAL}"
+    fi
     truncate -s "${FATSIZE}K" "${FATIMG}"
     mformat -v iPXE -i "${FATIMG}" ${FATARGS} ::
     mcopy -i "${FATIMG}" -s "${FATDIR}"/* ::