nixpkgs: afl 1.58b -> 1.63b

- Adds new LLVM-based instrumentation support via afl-clang-fast.
  - Experimental support for cgroup management via afl-cgroup, to
    mitigate OOM issues when using afl with address sanitizer.

Signed-off-by: Austin Seipp <aseipp@pobox.com>

pkgs/tools/security/afl/README.md

@@ -11,9 +11,9 @@ right QEMU version and options in `qemu.nix`:
 
 https://github.com/mirrorer/afl/blob/master/qemu_mode/build_qemu_support.sh
 
-`afl-config.h` and `afl-qemu-cpu-inl.h` are part of the afl source
-code, and copied from `config.h` and `afl-qemu-cpu-inl.h`
-appropriately. The QEMU patches need to be slightly adjusted to
-`#include` these files (the patches try to otherwise include files
-like `../../config.h` which causes the build to fail). See `qemu.nix`
-for details.
+`afl-config.h`, `afl-types.h`, and `afl-qemu-cpu-inl.h` are part of
+the afl source code, and copied from `config.h`, `types.h` and
+`afl-qemu-cpu-inl.h` appropriately. These files and the QEMU patches
+need to be slightly adjusted to fix their `#include`s (the patches
+try to otherwise include files like `../../config.h` which causes the
+build to fail).

pkgs/tools/security/afl/default.nix

@@ -1,4 +1,5 @@
-{ stdenv, fetchurl, bash, callPackage, makeWrapper }:
+{ stdenv, fetchurl, bash, callPackage, makeWrapper
+, clang, llvm, which, libcgroup }:
 
 let
   afl-qemu = callPackage ./qemu.nix {};
@@ -8,16 +9,21 @@ let
 in
 stdenv.mkDerivation rec {
   name    = "afl-${version}";
-  version = "1.58b";
+  version = "1.63b";
 
   src = fetchurl {
     url    = "http://lcamtuf.coredump.cx/afl/releases/${name}.tgz";
-    sha256 = "1szggm4x9i9bsrcb99s5vbgncagp7jvhz8cg9amkx7p6mp2x4pld";
+    sha256 = "1v3py0g52j687qacwhri8jbz2h0ggh3zqknp011z5ijf820vc09g";
   };
 
-  buildInputs  = [ makeWrapper ];
+  # Note: libcgroup isn't needed for building, just for the afl-cgroup
+  # script.
+  buildInputs  = [ makeWrapper clang llvm which ];
 
-  buildPhase   = "make PREFIX=$out";
+  buildPhase   = ''
+    make PREFIX=$out
+    cd llvm_mode && make && cd ..
+  '';
   installPhase = ''
     # Do the normal installation
     make install PREFIX=$out
@@ -25,6 +31,17 @@ stdenv.mkDerivation rec {
     # Install the custom QEMU emulator for binary blob fuzzing.
     cp ${afl-qemu}/bin/${qemu-exe-name} $out/bin/afl-qemu-trace
 
+    # Install the cgroups wrapper for asan-based fuzzing.
+    cp experimental/asan_cgroups/limit_memory.sh $out/bin/afl-cgroup
+    chmod +x $out/bin/afl-cgroup
+    substituteInPlace $out/bin/afl-cgroup \
+      --replace "cgcreate" "${libcgroup}/bin/cgcreate" \
+      --replace "cgexec"   "${libcgroup}/bin/cgexec" \
+      --replace "cgdelete" "${libcgroup}/bin/cgdelete"
+
+    # Patch shebangs before wrapping
+    patchShebangs $out/bin
+
     # Wrap every program with a custom $AFL_PATH; I believe there is a
     # bug in afl which causes it to fail to find `afl-qemu-trace`
     # relative to `afl-fuzz` or `afl-showmap`, so we instead set

pkgs/tools/security/afl/qemu-patches/afl-config.h

@@ -138,6 +138,11 @@
 
 #define TMIN_MAX_FILE       (10 * 1024 * 1024)
 
+/* Block normalization steps for afl-tmin: */
+
+#define TMIN_SET_MIN_SIZE   4
+#define TMIN_SET_STEPS      128
+
 /* Maximum dictionary token size (-x), in bytes: */
 
 #define MAX_DICT_FILE       128

pkgs/tools/security/afl/qemu-patches/afl-qemu-cpu-inl.h

@@ -134,6 +134,12 @@ static void afl_setup(void) {
 
     if (afl_area_ptr == (void*)-1) exit(1);
 
+    /* With AFL_INST_RATIO set to a low value, we want to touch the bitmap
+       so that the parent doesn't give up on us. */
+
+    if (inst_r) afl_area_ptr[0] = 1;
+
+
   }
 
   if (getenv("AFL_INST_LIBS")) {

pkgs/tools/security/afl/qemu.nix

@@ -65,7 +65,7 @@ stdenv.mkDerivation rec {
 
   meta = with stdenv.lib; {
     homepage = http://www.qemu.org/;
-    description = "Fork of QEMU with American Fuzzy Lop instrumentation support";
+    description = "Fork of QEMU with AFL instrumentation support";
     license = licenses.gpl2Plus;
     maintainers = with maintainers; [ thoughtpolice ];
     platforms = platforms.linux;