diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/oss-fuzz/build.sh | 14 | ||||
| -rwxr-xr-x | scripts/oss-fuzz/minimize_qtest_trace.py | 157 | ||||
| -rwxr-xr-x | scripts/oss-fuzz/reorder_fuzzer_qtest_trace.py | 103 | ||||
| -rwxr-xr-x[-rw-r--r--] | scripts/tracetool.py | 0 | ||||
| -rw-r--r-- | scripts/undefsym.py | 11 |
5 files changed, 278 insertions, 7 deletions
diff --git a/scripts/oss-fuzz/build.sh b/scripts/oss-fuzz/build.sh index 0c3ca9e06f..fcae4a0c26 100755 --- a/scripts/oss-fuzz/build.sh +++ b/scripts/oss-fuzz/build.sh @@ -62,6 +62,9 @@ fi mkdir -p "$DEST_DIR/lib/" # Copy the shared libraries here +mkdir -p "$DEST_DIR/bin/" # Copy executables that shouldn't + # be treated as fuzzers by oss-fuzz here + # Build once to get the list of dynamic lib paths, and copy them over ../configure --disable-werror --cc="$CC" --cxx="$CXX" --enable-fuzzing \ --prefix="$DEST_DIR" --bindir="$DEST_DIR" --datadir="$DEST_DIR/data/" \ @@ -88,13 +91,22 @@ make "-j$(nproc)" qemu-fuzz-i386 V=1 # Copy over the datadir cp -r ../pc-bios/ "$DEST_DIR/pc-bios" +cp "./qemu-fuzz-i386" "$DEST_DIR/bin/" + # Run the fuzzer with no arguments, to print the help-string and get the list # of available fuzz-targets. Copy over the qemu-fuzz-i386, naming it according # to each available fuzz target (See 05509c8e6d fuzz: select fuzz target using # executable name) for target in $(./qemu-fuzz-i386 | awk '$1 ~ /\*/ {print $2}'); do - cp qemu-fuzz-i386 "$DEST_DIR/qemu-fuzz-i386-target-$target" + # Ignore the generic-fuzz target, as it requires some environment variables + # to be configured. We have some generic-fuzz-{pc-q35, floppy, ...} targets + # that are thin wrappers around this target that set the required + # environment variables according to predefined configs. + if [ "$target" != "generic-fuzz" ]; then + ln "$DEST_DIR/bin/qemu-fuzz-i386" \ + "$DEST_DIR/qemu-fuzz-i386-target-$target" + fi done echo "Done. The fuzzers are located in $DEST_DIR" diff --git a/scripts/oss-fuzz/minimize_qtest_trace.py b/scripts/oss-fuzz/minimize_qtest_trace.py new file mode 100755 index 0000000000..5e405a0d5f --- /dev/null +++ b/scripts/oss-fuzz/minimize_qtest_trace.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +This takes a crashing qtest trace and tries to remove superflous operations +""" + +import sys +import os +import subprocess +import time +import struct + +QEMU_ARGS = None +QEMU_PATH = None +TIMEOUT = 5 +CRASH_TOKEN = None + +write_suffix_lookup = {"b": (1, "B"), + "w": (2, "H"), + "l": (4, "L"), + "q": (8, "Q")} + +def usage(): + sys.exit("""\ +Usage: QEMU_PATH="/path/to/qemu" QEMU_ARGS="args" {} input_trace output_trace +By default, will try to use the second-to-last line in the output to identify +whether the crash occred. Optionally, manually set a string that idenitifes the +crash by setting CRASH_TOKEN= +""".format((sys.argv[0]))) + +def check_if_trace_crashes(trace, path): + global CRASH_TOKEN + with open(path, "w") as tracefile: + tracefile.write("".join(trace)) + + rc = subprocess.Popen("timeout -s 9 {timeout}s {qemu_path} {qemu_args} 2>&1\ + < {trace_path}".format(timeout=TIMEOUT, + qemu_path=QEMU_PATH, + qemu_args=QEMU_ARGS, + trace_path=path), + shell=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + stdo = rc.communicate()[0] + output = stdo.decode('unicode_escape') + if rc.returncode == 137: # Timed Out + return False + if len(output.splitlines()) < 2: + return False + + if CRASH_TOKEN is None: + CRASH_TOKEN = output.splitlines()[-2] + + return CRASH_TOKEN in output + + +def minimize_trace(inpath, outpath): + global TIMEOUT + with open(inpath) as f: + trace = f.readlines() + start = time.time() + if not check_if_trace_crashes(trace, outpath): + sys.exit("The input qtest trace didn't cause a crash...") + end = time.time() + print("Crashed in {} seconds".format(end-start)) + TIMEOUT = (end-start)*5 + print("Setting the timeout for {} seconds".format(TIMEOUT)) + print("Identifying Crashes by this string: {}".format(CRASH_TOKEN)) + + i = 0 + newtrace = trace[:] + # For each line + while i < len(newtrace): + # 1.) Try to remove it completely and reproduce the crash. If it works, + # we're done. + prior = newtrace[i] + print("Trying to remove {}".format(newtrace[i])) + # Try to remove the line completely + newtrace[i] = "" + if check_if_trace_crashes(newtrace, outpath): + i += 1 + continue + newtrace[i] = prior + + # 2.) Try to replace write{bwlq} commands with a write addr, len + # command. Since this can require swapping endianness, try both LE and + # BE options. We do this, so we can "trim" the writes in (3) + if (newtrace[i].startswith("write") and not + newtrace[i].startswith("write ")): + suffix = newtrace[i].split()[0][-1] + assert(suffix in write_suffix_lookup) + addr = int(newtrace[i].split()[1], 16) + value = int(newtrace[i].split()[2], 16) + for endianness in ['<', '>']: + data = struct.pack("{end}{size}".format(end=endianness, + size=write_suffix_lookup[suffix][1]), + value) + newtrace[i] = "write {addr} {size} 0x{data}\n".format( + addr=hex(addr), + size=hex(write_suffix_lookup[suffix][0]), + data=data.hex()) + if(check_if_trace_crashes(newtrace, outpath)): + break + else: + newtrace[i] = prior + + # 3.) If it is a qtest write command: write addr len data, try to split + # it into two separate write commands. If splitting the write down the + # middle does not work, try to move the pivot "left" and retry, until + # there is no space left. The idea is to prune unneccessary bytes from + # long writes, while accommodating arbitrary MemoryRegion access sizes + # and alignments. + if newtrace[i].startswith("write "): + addr = int(newtrace[i].split()[1], 16) + length = int(newtrace[i].split()[2], 16) + data = newtrace[i].split()[3][2:] + if length > 1: + leftlength = int(length/2) + rightlength = length - leftlength + newtrace.insert(i+1, "") + while leftlength > 0: + newtrace[i] = "write {addr} {size} 0x{data}\n".format( + addr=hex(addr), + size=hex(leftlength), + data=data[:leftlength*2]) + newtrace[i+1] = "write {addr} {size} 0x{data}\n".format( + addr=hex(addr+leftlength), + size=hex(rightlength), + data=data[leftlength*2:]) + if check_if_trace_crashes(newtrace, outpath): + break + else: + leftlength -= 1 + rightlength += 1 + if check_if_trace_crashes(newtrace, outpath): + i -= 1 + else: + newtrace[i] = prior + del newtrace[i+1] + i += 1 + check_if_trace_crashes(newtrace, outpath) + + +if __name__ == '__main__': + if len(sys.argv) < 3: + usage() + + QEMU_PATH = os.getenv("QEMU_PATH") + QEMU_ARGS = os.getenv("QEMU_ARGS") + if QEMU_PATH is None or QEMU_ARGS is None: + usage() + # if "accel" not in QEMU_ARGS: + # QEMU_ARGS += " -accel qtest" + CRASH_TOKEN = os.getenv("CRASH_TOKEN") + QEMU_ARGS += " -qtest stdio -monitor none -serial none " + minimize_trace(sys.argv[1], sys.argv[2]) diff --git a/scripts/oss-fuzz/reorder_fuzzer_qtest_trace.py b/scripts/oss-fuzz/reorder_fuzzer_qtest_trace.py new file mode 100755 index 0000000000..890e1def85 --- /dev/null +++ b/scripts/oss-fuzz/reorder_fuzzer_qtest_trace.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Use this to convert qtest log info from a generic fuzzer input into a qtest +trace that you can feed into a standard qemu-system process. Example usage: + +QEMU_FUZZ_ARGS="-machine q35,accel=qtest" QEMU_FUZZ_OBJECTS="*" \ + ./i386-softmmu/qemu-fuzz-i386 --fuzz-target=generic-pci-fuzz +# .. Finds some crash +QTEST_LOG=1 FUZZ_SERIALIZE_QTEST=1 \ +QEMU_FUZZ_ARGS="-machine q35,accel=qtest" QEMU_FUZZ_OBJECTS="*" \ + ./i386-softmmu/qemu-fuzz-i386 --fuzz-target=generic-pci-fuzz + /path/to/crash 2> qtest_log_output +scripts/oss-fuzz/reorder_fuzzer_qtest_trace.py qtest_log_output > qtest_trace +./i386-softmmu/qemu-fuzz-i386 -machine q35,accel=qtest \ + -qtest stdin < qtest_trace + +### Details ### + +Some fuzzer make use of hooks that allow us to populate some memory range, just +before a DMA read from that range. This means that the fuzzer can produce +activity that looks like: + [start] read from mmio addr + [end] read from mmio addr + [start] write to pio addr + [start] fill a DMA buffer just in time + [end] fill a DMA buffer just in time + [start] fill a DMA buffer just in time + [end] fill a DMA buffer just in time + [end] write to pio addr + [start] read from mmio addr + [end] read from mmio addr + +We annotate these "nested" DMA writes, so with QTEST_LOG=1 the QTest trace +might look something like: +[R +0.028431] readw 0x10000 +[R +0.028434] outl 0xc000 0xbeef # Triggers a DMA read from 0xbeef and 0xbf00 +[DMA][R +0.034639] write 0xbeef 0x2 0xAAAA +[DMA][R +0.034639] write 0xbf00 0x2 0xBBBB +[R +0.028431] readw 0xfc000 + +This script would reorder the above trace so it becomes: +readw 0x10000 +write 0xbeef 0x2 0xAAAA +write 0xbf00 0x2 0xBBBB +outl 0xc000 0xbeef +readw 0xfc000 + +I.e. by the time, 0xc000 tries to read from DMA, those DMA buffers have already +been set up, removing the need for the DMA hooks. We can simply provide this +reordered trace via -qtest stdio to reproduce the input + +Note: this won't work for traces where the device tries to read from the same +DMA region twice in between MMIO/PIO commands. E.g: + [R +0.028434] outl 0xc000 0xbeef + [DMA][R +0.034639] write 0xbeef 0x2 0xAAAA + [DMA][R +0.034639] write 0xbeef 0x2 0xBBBB + +The fuzzer will annotate suspected double-fetches with [DOUBLE-FETCH]. This +script looks for these tags and warns the users that the resulting trace might +not reproduce the bug. +""" + +import sys + +__author__ = "Alexander Bulekov <alxndr@bu.edu>" +__copyright__ = "Copyright (C) 2020, Red Hat, Inc." +__license__ = "GPL version 2 or (at your option) any later version" + +__maintainer__ = "Alexander Bulekov" +__email__ = "alxndr@bu.edu" + + +def usage(): + sys.exit("Usage: {} /path/to/qtest_log_output".format((sys.argv[0]))) + + +def main(filename): + with open(filename, "r") as f: + trace = f.readlines() + + # Leave only lines that look like logged qtest commands + trace[:] = [x.strip() for x in trace if "[R +" in x + or "[S +" in x and "CLOSED" not in x] + + for i in range(len(trace)): + if i+1 < len(trace): + if "[DMA]" in trace[i+1]: + if "[DOUBLE-FETCH]" in trace[i+1]: + sys.stderr.write("Warning: Likely double fetch on line" + "{}.\n There will likely be problems " + "reproducing behavior with the " + "resulting qtest trace\n\n".format(i+1)) + trace[i], trace[i+1] = trace[i+1], trace[i] + for line in trace: + print(line.split("]")[-1].strip()) + + +if __name__ == '__main__': + if len(sys.argv) == 1: + usage() + main(sys.argv[1]) diff --git a/scripts/tracetool.py b/scripts/tracetool.py index 31146242b7..31146242b7 100644..100755 --- a/scripts/tracetool.py +++ b/scripts/tracetool.py diff --git a/scripts/undefsym.py b/scripts/undefsym.py index 69a895cd26..4b6a72d95f 100644 --- a/scripts/undefsym.py +++ b/scripts/undefsym.py @@ -15,12 +15,11 @@ def filter_lines_set(stdout, from_staticlib): linesSet = set() for line in stdout.splitlines(): tokens = line.split(b' ') - if len(tokens) >= 1: - if len(tokens) > 1: - if from_staticlib and tokens[1] == b'U': - continue - if not from_staticlib and tokens[1] != b'U': - continue + if len(tokens) >= 2: + if from_staticlib and tokens[1] == b'U': + continue + if not from_staticlib and tokens[1] != b'U': + continue new_line = b'-Wl,-u,' + tokens[0] if not new_line in linesSet: linesSet.add(new_line) |