target/i386: fix fbstp handling of out-of-range values

The fbstp implementation fails to check for out-of-range and invalid
values, instead just taking the result of conversion to int64_t and
storing its sign and low 18 decimal digits.  Fix this by checking for
an out-of-range result (invalid conversions always result in INT64_MAX
or INT64_MIN from the softfloat code, which are large enough to be
considered as out-of-range by this code) and storing the packed BCD
indefinite encoding in that case.

Signed-off-by: Joseph Myers <joseph@codesourcery.com>
Message-Id: <alpine.DEB.2.21.2005132351110.11687@digraph.polyomino.org.uk>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

1 files changed, 115 insertions, 0 deletions

diff --git a/tests/tcg/i386/test-i386-fbstp.c b/tests/tcg/i386/test-i386-fbstp.c
index d368949188..73bf56b9dc 100644
--- a/tests/tcg/i386/test-i386-fbstp.c
+++ b/tests/tcg/i386/test-i386-fbstp.c
@@ -1,8 +1,19 @@
 /* Test fbstp instruction.  */
 
+#include <stdint.h>
 #include <stdio.h>
 #include <string.h>
 
+union u {
+    struct { uint64_t sig; uint16_t sign_exp; } s;
+    long double ld;
+};
+
+volatile union u ld_invalid_1 = { .s = { 1, 1234 } };
+volatile union u ld_invalid_2 = { .s = { 0, 1234 } };
+volatile union u ld_invalid_3 = { .s = { 0, 0x7fff } };
+volatile union u ld_invalid_4 = { .s = { (UINT64_C(1) << 63) - 1, 0x7fff } };
+
 int main(void)
 {
     int ret = 0;
@@ -21,5 +32,109 @@ int main(void)
         printf("FAIL: fbstp -0.1\n");
         ret = 1;
     }
+    memset(out, 0x1f, sizeof out);
+    __asm__ volatile ("fbstp %0" : "=m" (out) : "t" (-987654321987654321.0L) :
+                      "st");
+    out[9] &= 0x80;
+    if (memcmp(out, "\x21\x43\x65\x87\x19\x32\x54\x76\x98\x80",
+               sizeof out) != 0) {
+        printf("FAIL: fbstp -987654321987654321\n");
+        ret = 1;
+    }
+    memset(out, 0x12, sizeof out);
+    __asm__ volatile ("fbstp %0" : "=m" (out) : "t" (999999999999999999.5L) :
+                      "st");
+    if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+        printf("FAIL: fbstp 999999999999999999.5\n");
+        ret = 1;
+    }
+    memset(out, 0x12, sizeof out);
+    __asm__ volatile ("fbstp %0" : "=m" (out) : "t" (1000000000000000000.0L) :
+                      "st");
+    if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+        printf("FAIL: fbstp 1000000000000000000\n");
+        ret = 1;
+    }
+    memset(out, 0x12, sizeof out);
+    __asm__ volatile ("fbstp %0" : "=m" (out) : "t" (1e30L) : "st");
+    if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+        printf("FAIL: fbstp 1e30\n");
+        ret = 1;
+    }
+    memset(out, 0x12, sizeof out);
+    __asm__ volatile ("fbstp %0" : "=m" (out) : "t" (-999999999999999999.5L) :
+                      "st");
+    if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+        printf("FAIL: fbstp -999999999999999999.5\n");
+        ret = 1;
+    }
+    memset(out, 0x12, sizeof out);
+    __asm__ volatile ("fbstp %0" : "=m" (out) : "t" (-1000000000000000000.0L) :
+                      "st");
+    if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+        printf("FAIL: fbstp -1000000000000000000\n");
+        ret = 1;
+    }
+    memset(out, 0x12, sizeof out);
+    __asm__ volatile ("fbstp %0" : "=m" (out) : "t" (-1e30L) : "st");
+    if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+        printf("FAIL: fbstp -1e30\n");
+        ret = 1;
+    }
+    memset(out, 0x12, sizeof out);
+    __asm__ volatile ("fbstp %0" : "=m" (out) : "t" (__builtin_infl()) : "st");
+    if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+        printf("FAIL: fbstp inf\n");
+        ret = 1;
+    }
+    memset(out, 0x12, sizeof out);
+    __asm__ volatile ("fbstp %0" : "=m" (out) : "t" (-__builtin_infl()) :
+                      "st");
+    if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+        printf("FAIL: fbstp -inf\n");
+        ret = 1;
+    }
+    memset(out, 0x12, sizeof out);
+    __asm__ volatile ("fbstp %0" : "=m" (out) : "t" (__builtin_nanl("")) :
+                      "st");
+    if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+        printf("FAIL: fbstp nan\n");
+        ret = 1;
+    }
+    memset(out, 0x12, sizeof out);
+    __asm__ volatile ("fbstp %0" : "=m" (out) : "t" (-__builtin_nanl("")) :
+                      "st");
+    if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+        printf("FAIL: fbstp -nan\n");
+        ret = 1;
+    }
+    memset(out, 0x12, sizeof out);
+    __asm__ volatile ("fbstp %0" : "=m" (out) : "t" (ld_invalid_1.ld) :
+                      "st");
+    if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+        printf("FAIL: fbstp invalid 1\n");
+        ret = 1;
+    }
+    memset(out, 0x12, sizeof out);
+    __asm__ volatile ("fbstp %0" : "=m" (out) : "t" (ld_invalid_2.ld) :
+                      "st");
+    if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+        printf("FAIL: fbstp invalid 2\n");
+        ret = 1;
+    }
+    memset(out, 0x12, sizeof out);
+    __asm__ volatile ("fbstp %0" : "=m" (out) : "t" (ld_invalid_3.ld) :
+                      "st");
+    if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+        printf("FAIL: fbstp invalid 3\n");
+        ret = 1;
+    }
+    memset(out, 0x12, sizeof out);
+    __asm__ volatile ("fbstp %0" : "=m" (out) : "t" (ld_invalid_4.ld) :
+                      "st");
+    if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+        printf("FAIL: fbstp invalid 4\n");
+        ret = 1;
+    }
     return ret;
 }