[uri] Retain original encodings for path, query, and fragment fields

iPXE decodes any percent-encoded characters during the URI parsing
stage, thereby allowing protocol implementations to consume the raw
field values directly without further decoding.

When reconstructing a URI string for use in an HTTP request line, the
percent-encoding is currently reapplied in a reversible way: we
guarantee that our reconstructed URI string could be decoded to give
the same raw field values.

This technically violates RFC3986, which states that "URIs that differ
in the replacement of a reserved character with its corresponding
percent-encoded octet are not equivalent".  Experiments show that
several HTTP server applications will attach meaning to the choice of
whether or not a particular character was percent-encoded, even when
the percent-encoding is unnecessary from the perspective of parsing
the URI into its component fields.

Fix by storing the originally encoded substrings for the path, query,
and fragment fields and using these original encoded versions when
reconstructing a URI string.  The path field is also stored as a
decoded string, for use by protocols such as TFTP that communicate
using raw strings rather than URI-encoded strings.  All other fields
(such as the username and password) continue to be stored only in
their decoded versions since nothing ever needs to know the originally
encoded versions of these fields.

Signed-off-by: Michael Brown <mcb30@ipxe.org>

1 files changed, 40 insertions, 13 deletions

diff --git a/src/tests/uri_test.c b/src/tests/uri_test.c
index 92c2f903..929ab363 100644
--- a/src/tests/uri_test.c
+++ b/src/tests/uri_test.c
@@ -149,8 +149,10 @@ static void uri_okx ( struct uri *uri, struct uri *expected, const char *file,
 	okx ( uristrcmp ( uri->host, expected->host ) == 0, file, line );
 	okx ( uristrcmp ( uri->port, expected->port ) == 0, file, line );
 	okx ( uristrcmp ( uri->path, expected->path ) == 0, file, line );
-	okx ( uristrcmp ( uri->query, expected->query ) == 0, file, line );
-	okx ( uristrcmp ( uri->fragment, expected->fragment ) == 0, file, line);
+	okx ( uristrcmp ( uri->epath, expected->epath ) == 0, file, line );
+	okx ( uristrcmp ( uri->equery, expected->equery ) == 0, file, line );
+	okx ( uristrcmp ( uri->efragment, expected->efragment ) == 0,
+	      file, line);
 	okx ( uri->params == expected->params, file, line );
 }
 #define uri_ok( uri, expected ) uri_okx ( uri, expected, __FILE__, __LINE__ )
@@ -490,25 +492,33 @@ static struct uri_test uri_empty = {
 /** Basic HTTP URI */
 static struct uri_test uri_boot_ipxe_org = {
 	"http://boot.ipxe.org/demo/boot.php",
-	{ .scheme = "http", .host = "boot.ipxe.org", .path = "/demo/boot.php" }
+	{ .scheme = "http", .host = "boot.ipxe.org",
+	  .path = "/demo/boot.php", .epath = "/demo/boot.php" },
 };
 
 /** Basic opaque URI */
 static struct uri_test uri_mailto = {
 	"mailto:ipxe-devel@lists.ipxe.org",
-	{ .scheme = "mailto", .opaque = "ipxe-devel@lists.ipxe.org" }
+	{ .scheme = "mailto", .opaque = "ipxe-devel@lists.ipxe.org" },
+};
+
+/** Basic host-only URI */
+static struct uri_test uri_host = {
+	"http://boot.ipxe.org",
+	{ .scheme = "http", .host = "boot.ipxe.org" },
 };
 
 /** Basic path-only URI */
 static struct uri_test uri_path = {
 	"/var/lib/tftpboot/pxelinux.0",
-	{ .path = "/var/lib/tftpboot/pxelinux.0" },
+	{ .path = "/var/lib/tftpboot/pxelinux.0",
+	  .epath ="/var/lib/tftpboot/pxelinux.0" },
 };
 
 /** Path-only URI with escaped characters */
 static struct uri_test uri_path_escaped = {
 	"/hello%20world%3F",
-	{ .path = "/hello world?" },
+	{ .path = "/hello world?", .epath = "/hello%20world%3F" },
 };
 
 /** HTTP URI with all the trimmings */
@@ -521,8 +531,9 @@ static struct uri_test uri_http_all = {
 		.host = "example.com",
 		.port = "3001",
 		.path = "/~foo/cgi-bin/foo.pl",
-		.query = "a=b&c=d",
-		.fragment = "bit",
+		.epath = "/~foo/cgi-bin/foo.pl",
+		.equery = "a=b&c=d",
+		.efragment = "bit",
 	},
 };
 
@@ -533,8 +544,9 @@ static struct uri_test uri_http_escaped = {
 		.scheme = "https",
 		.host = "test.ipxe.org",
 		.path = "/wtf?\n",
-		.query = "kind#of/uri is",
-		.fragment = "this?",
+		.epath = "/wtf%3F%0A",
+		.equery = "kind%23of/uri%20is",
+		.efragment = "this%3F",
 	},
 };
 
@@ -550,8 +562,9 @@ static struct uri_test uri_http_escaped_improper = {
 		.scheme = "https",
 		.host = "test.ipxe.org",
 		.path = "/wtf?\n",
-		.query = "kind#of/uri is",
-		.fragment = "this?",
+		.epath = "/wt%66%3f\n",
+		.equery = "kind%23of/uri is",
+		.efragment = "this?",
 	},
 };
 
@@ -562,6 +575,7 @@ static struct uri_test uri_ipv6 = {
 		.scheme = "http",
 		.host = "[2001:ba8:0:1d4::6950:5845]",
 		.path = "/",
+		.epath = "/",
 	},
 };
 
@@ -573,6 +587,7 @@ static struct uri_test uri_ipv6_port = {
 		.host = "[2001:ba8:0:1d4::6950:5845]",
 		.port = "8001",
 		.path = "/boot",
+		.epath = "/boot",
 	},
 };
 
@@ -583,6 +598,7 @@ static struct uri_test uri_ipv6_local = {
 		.scheme = "http",
 		.host = "[fe80::69ff:fe50:5845%net0]",
 		.path = "/ipxe",
+		.epath = "/ipxe",
 	},
 };
 
@@ -598,6 +614,7 @@ static struct uri_test uri_ipv6_local_non_conforming = {
 		.scheme = "http",
 		.host = "[fe80::69ff:fe50:5845%net0]",
 		.path = "/ipxe",
+		.epath = "/ipxe",
 	},
 };
 
@@ -625,6 +642,7 @@ static struct uri_test uri_file_absolute = {
 	{
 		.scheme = "file",
 		.path = "/boot/script.ipxe",
+		.epath = "/boot/script.ipxe",
 	},
 };
 
@@ -635,6 +653,7 @@ static struct uri_test uri_file_volume = {
 		.scheme = "file",
 		.host = "hpilo",
 		.path = "/boot/script.ipxe",
+		.epath = "/boot/script.ipxe",
 	},
 };
 
@@ -736,6 +755,7 @@ static struct uri_pxe_test uri_pxe_absolute = {
 		.scheme = "http",
 		.host = "not.a.tftp",
 		.path = "/uri",
+		.epath = "/uri",
 	},
 	"http://not.a.tftp/uri",
 };
@@ -754,6 +774,7 @@ static struct uri_pxe_test uri_pxe_absolute_path = {
 		.scheme = "tftp",
 		.host = "192.168.0.2",
 		.path = "//absolute/path",
+		.epath = "//absolute/path",
 	},
 	"tftp://192.168.0.2//absolute/path",
 };
@@ -772,6 +793,7 @@ static struct uri_pxe_test uri_pxe_relative_path = {
 		.scheme = "tftp",
 		.host = "192.168.0.3",
 		.path = "/relative/path",
+		.epath = "/relative/path",
 	},
 	"tftp://192.168.0.3/relative/path",
 };
@@ -790,8 +812,9 @@ static struct uri_pxe_test uri_pxe_icky = {
 		.scheme = "tftp",
 		.host = "10.0.0.6",
 		.path = "/C:\\tftpboot\\icky#path",
+		.epath = "/C:\\tftpboot\\icky#path",
 	},
-	"tftp://10.0.0.6/C%3A\\tftpboot\\icky%23path",
+	"tftp://10.0.0.6/C:\\tftpboot\\icky#path",
 };
 
 /** PXE URI with custom port */
@@ -810,6 +833,7 @@ static struct uri_pxe_test uri_pxe_port = {
 		.host = "192.168.0.1",
 		.port = "4069",
 		.path = "//another/path",
+		.epath = "//another/path",
 	},
 	"tftp://192.168.0.1:4069//another/path",
 };
@@ -873,6 +897,7 @@ static struct uri_params_test uri_params = {
 		.scheme = "http",
 		.host = "boot.ipxe.org",
 		.path = "/demo/boot.php",
+		.epath = "/demo/boot.php",
 	},
 	NULL,
 	uri_params_list,
@@ -902,6 +927,7 @@ static struct uri_params_test uri_named_params = {
 		.host = "192.168.100.4",
 		.port = "3001",
 		.path = "/register",
+		.epath = "/register",
 	},
 	"foo",
 	uri_named_params_list,
@@ -917,6 +943,7 @@ static void uri_test_exec ( void ) {
 	uri_parse_format_dup_ok ( &uri_empty );
 	uri_parse_format_dup_ok ( &uri_boot_ipxe_org );
 	uri_parse_format_dup_ok ( &uri_mailto );
+	uri_parse_format_dup_ok ( &uri_host );
 	uri_parse_format_dup_ok ( &uri_path );
 	uri_parse_format_dup_ok ( &uri_path_escaped );
 	uri_parse_format_dup_ok ( &uri_http_all );