Cryptofuzz ci

tests/ci/cdk/cdk/codebuild/github_ci_fuzzing_omnibus.yaml

@@ -22,3 +22,19 @@ batch:
         privileged-mode: true
         compute-type: BUILD_GENERAL1_LARGE
         image: ARM_ECR_REPO_PLACEHOLDER:ubuntu-20.04_clang-10x_latest
+
+    - identifier: ubuntu2004_clang10_x86_64_cryptofuzz
+      buildspec: ./tests/ci/codebuild/linux-x86/run_cryptofuzz.yml
+      env:
+        type: LINUX_CONTAINER
+        privileged-mode: true
+        compute-type: BUILD_GENERAL1_LARGE
+        image: X86_ECR_REPO_PLACEHOLDER:ubuntu-20.04_cryptofuzz_latest
+
+    - identifier: ubuntu2004_clang10_arm_cryptofuzz
+      buildspec: ./tests/ci/codebuild/linux-x86/run_cryptofuzz.yml
+      env:
+        type: ARM_CONTAINER
+        privileged-mode: true
+        compute-type: BUILD_GENERAL1_LARGE
+        image: ARM_ECR_REPO_PLACEHOLDER:ubuntu-20.04_cryptofuzz_latest

tests/ci/cdk/setup.py

@@ -20,10 +20,10 @@
 
     install_requires=[
         # CDK dependencies.
-        "aws-cdk.core==1.64.0",
-        "aws-cdk.aws-codebuild==1.64.0",
-        "aws-cdk.aws-ecr==1.64.0",
-        "aws-cdk.aws-iam==1.64.0",
+        "aws-cdk.core==1.97.0",
+        "aws-cdk.aws-codebuild==1.97.0",
+        "aws-cdk.aws-ecr==1.97.0",
+        "aws-cdk.aws-iam==1.97.0",
         # PyYAML is a YAML parser and emitter for Python. Used to read build_spec.yaml.
         "pyyaml==5.3.1",
         # A formatter for Python code.

tests/ci/codebuild/linux-x86/run_cryptofuzz.yml

@@ -0,0 +1,14 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+version: 0.2
+
+phases:
+  pre_build:
+    commands:
+      # To use this build spec file, CMake environment variable CC and CXX compiler should be defined before build.
+      - if [[ -z "${CC+x}" || -z "${CC}" ]]; then echo "CC is not defined." && exit 1; else ${CC} --version && echo "Found correct CC."; fi
+      - if [[ -z "${CXX+x}" || -z "${CXX}" ]]; then echo "CXX is not defined." && exit 1; else ${CXX} --version && echo "Found correct CXX."; fi
+  build:
+    commands:
+      - ./tests/ci/run_cryptofuzz.sh

tests/ci/common_fuzz.sh

@@ -18,6 +18,14 @@ else
 fi
 echo "$BUILD_ID"
 
+PLATFORM=$(uname -m)
+DATE_NOW="$(date +%Y-%m-%d)"
+FAILURE_ROOT="${CORPUS_ROOT}/runs/${DATE_NOW}/${BUILD_ID}"
+ALL_RUN_ROOT="${BUILD_ROOT}/fuzz_run_root"
+rm -rf "$ALL_RUN_ROOT"
+
+
+
 function put_metric_count {
   put_metric --unit Count "$@"
 }
@@ -31,3 +39,90 @@ function put_metric {
   # Turn it back on for the rest of the build
   set -e
 }
+
+function run_fuzz_test {
+  SHARED_CORPUS="${CORPUS_ROOT}/shared_corpus/${FUZZ_NAME}/shared_corpus"
+  FUZZ_TEST_ROOT="${ALL_RUN_ROOT}/${FUZZ_NAME}"
+  FUZZ_TEST_CORPUS="${FUZZ_TEST_ROOT}/run_corpus"
+  ARTIFACTS_FOLDER="${FUZZ_TEST_ROOT}/artifacts"
+  FUZZ_RUN_LOGS="${FUZZ_TEST_ROOT}/logs"
+  SUMMARY_LOG="${FUZZ_RUN_LOGS}/summary.log"
+  mkdir -p "$SHARED_CORPUS" "$FUZZ_TEST_ROOT" "$FUZZ_TEST_CORPUS" "$ARTIFACTS_FOLDER" "$FUZZ_RUN_LOGS"
+
+
+    # Calculate starting metrics and post to CloudWatch
+  ORIGINAL_SHARED_CORPUS_FILE_COUNT=$(find "$SHARED_CORPUS" -type f | wc -l)
+  put_metric_count --metric-name SharedCorpusFileCount --value "$ORIGINAL_SHARED_CORPUS_FILE_COUNT" --dimensions "FuzzTest=$FUZZ_NAME"
+
+  # Perform the actual fuzzing!
+  # Step 1 run each fuzz test for the determined time. This will use the existing shared corpus (in EFS) and any files
+  # checked into the GitHub corpus. This runs the fuzzer with three folders: the first folder is where new inputs will
+  # go (FUZZ_TEST_CORPUS), all other folders will be used as input (SHARED_CORPUS and SRC_CORPUS). It will write new
+  # files to the temporary run corpus.
+  # https://llvm.org/docs/LibFuzzer.html#options
+  #
+  # Run with NUM_CPU_THREADS which will be physical cores on ARM and virtualized cores on x86 with hyper threading.
+  # Looking at the overall system fuzz rate running 1:1 with virtualized cores provides a noticeable speed up. This
+  # is slightly different than libfuzzer's recommendation of #cores/2.
+  # This could fail and we want to capture that (+e)
+  set +e
+  FUZZ_RUN_FAILURE=0
+  time "./${FUZZ_TEST_PATH}" -print_final_stats=1 -timeout="$FUZZ_TEST_TIMEOUT" -max_total_time="$TIME_FOR_EACH_FUZZ" \
+    -jobs="$NUM_CPU_THREADS" -workers="$NUM_CPU_THREADS" \
+    -artifact_prefix="$ARTIFACTS_FOLDER/" \
+    "$FUZZ_TEST_CORPUS" "$SHARED_CORPUS" "$SRC_CORPUS" 2>&1 | tee "$SUMMARY_LOG"
+  # This gets the status of the fuzz run which determines if we want to fail the build or not, otherwise we'd get the results of tee
+  if [ "${PIPESTATUS[0]}" == 1 ]; then
+    FUZZ_RUN_FAILURE=1
+  fi
+
+  # The libfuzzer logs are written to the current working directory and need to be moved after the test is done
+  mv ./*.log  "${FUZZ_RUN_LOGS}/."
+
+  if [ "$FUZZ_RUN_FAILURE" == 1 ]; then
+    FUZZ_TEST_FAILURE_ROOT="${FAILURE_ROOT}/${FUZZ_NAME}"
+    mkdir -p "$FUZZ_TEST_FAILURE_ROOT"
+
+    if [[ "$FUZZ_NAME" == "cryptofuzz" ]]; then
+      for ARTIFACT in "$ARTIFACTS_FOLDER"/*; do
+        ARTIFACT_NAME=$(basename "$ARTIFACT")
+        "./${FUZZ_TEST_PATH}" --debug "$ARTIFACT" | tee "${FUZZ_RUN_LOGS}/${ARTIFACT_NAME}.log"
+      done
+    fi
+
+    cp -r "$FUZZ_TEST_ROOT" "$FAILURE_ROOT"
+    cp "$FUZZ_TEST_PATH" "${FUZZ_TEST_FAILURE_ROOT}/${FUZZ_NAME}"
+
+    # If this fuzz run has failed the below metrics wont make a lot of sense, it could fail on the first input and publish a TestCount of 1 which makes all the metrics look weird
+    echo "${FUZZ_NAME} failed, see the above output for details. For all the logs see ${FAILURE_ROOT} in EFS"
+    exit 1
+  else
+    echo "Fuzz test ${FUZZ_NAME} finished successfully, not copying run logs and run corpus"
+  fi
+
+  set -e
+
+  # Step 2 merge any new files from the run corpus and GitHub src corpus into the shared corpus (EFS)
+  time "./${FUZZ_TEST_PATH}" -merge=1 "$SHARED_CORPUS" "$FUZZ_TEST_CORPUS" "$SRC_CORPUS"
+
+  # Calculate interesting metrics and post results to CloudWatch
+  FINAL_SHARED_CORPUS_FILE_COUNT=$(find "$SHARED_CORPUS" -type f | wc -l)
+  put_metric_count --metric-name SharedCorpusFileCount --value "$FINAL_SHARED_CORPUS_FILE_COUNT" --dimensions "FuzzTest=$FUZZ_NAME"
+
+  NEW_FUZZ_FILES=$(find "$FUZZ_TEST_CORPUS" -type f | wc -l)
+  put_metric_count --metric-name RunCorpusFileCount --value "$NEW_FUZZ_FILES" --dimensions "FuzzTest=$FUZZ_NAME,Platform=$PLATFORM"
+
+  TEST_COUNT=$(grep -o "stat::number_of_executed_units: [0-9]*" "$SUMMARY_LOG" | awk '{test_count += $2} END {print test_count}')
+  put_metric_count --metric-name TestCount --value "$TEST_COUNT" --dimensions "FuzzTest=$FUZZ_NAME,Platform=$PLATFORM"
+
+  TESTS_PER_SECOND=$((TEST_COUNT/TIME_FOR_EACH_FUZZ))
+  put_metric --metric-name TestRate --value "$TESTS_PER_SECOND" --unit Count/Second --dimensions "FuzzTest=$FUZZ_NAME,Platform=$PLATFORM"
+
+  FEATURE_COVERAGE=$(grep -o "ft: [0-9]*" "$SUMMARY_LOG" | awk '{print $2}' | sort | tail -1)
+  put_metric_count --metric-name FeatureCoverage --value "$FEATURE_COVERAGE" --dimensions "FuzzTest=$FUZZ_NAME,Platform=$PLATFORM"
+
+  BLOCK_COVERAGE=$(grep -o "cov: [0-9]*" "$SUMMARY_LOG" | awk '{print $2}' | sort | tail -1)
+  put_metric_count --metric-name BlockCoverage --value "$BLOCK_COVERAGE" --dimensions "FuzzTest=$FUZZ_NAME,Platform=$PLATFORM"
+
+  echo "${FUZZ_NAME} starting shared ${ORIGINAL_SHARED_CORPUS_FILE_COUNT} final shared ${FINAL_SHARED_CORPUS_FILE_COUNT} new files ${NEW_FUZZ_FILES} total test count ${TEST_COUNT} test rate ${TESTS_PER_SECOND} code coverage ${BLOCK_COVERAGE} feature coverage ${FEATURE_COVERAGE}"
+}

tests/ci/docker_images/build_cryptofuzz_modules.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+set -exo pipefail
+
+function env {
+  export "$1"="$2"
+  echo "export ${1}=\"${2}\"" >> "${FUZZ_ROOT}/fuzz_env.sh"
+}
+# Recommended flags from https://github.com/guidovranken/cryptofuzz/blob/master/docs/building.md
+export CFLAGS="-fsanitize=address,undefined,fuzzer-no-link -O2 -g"
+export CXXFLAGS="-fsanitize=address,undefined,fuzzer-no-link -D_GLIBCXX_DEBUG -O2 -g"
+
+# Setup base of Cryptofuzz
+cd "$FUZZ_ROOT"
+MODULES_ROOT="${FUZZ_ROOT}/modules"
+#git clone --depth 1 https://github.com/guidovranken/cryptofuzz.git
+git clone https://github.com/guidovranken/cryptofuzz.git
+cd cryptofuzz
+git checkout 76ffeff944403cdd840f06b8fc42e131e6258f36
+git rev-parse HEAD
+CRYPTOFUZZ_SRC=$(pwd)
+python3 gen_repository.py
+
+mkdir "$MODULES_ROOT"
+cd "$MODULES_ROOT"
+
+# Setup the other crypto libraries for differential fuzzing
+# Botan https://github.com/guidovranken/cryptofuzz/blob/master/docs/botan.md
+git clone --depth 1 https://github.com/randombit/botan.git
+cd botan
+git rev-parse HEAD
+python3 configure.py --cc-bin=$CXX --cc-abi-flags="$CXXFLAGS" --disable-shared --disable-modules=locking_allocator,x509,tls --build-targets=static --without-documentation
+make -j$(nproc)
+export CXXFLAGS="$CXXFLAGS -DCRYPTOFUZZ_BOTAN"
+env LIBBOTAN_A_PATH `realpath libbotan-3.a`
+env BOTAN_INCLUDE_PATH `realpath build/include`
+cd "${CRYPTOFUZZ_SRC}/modules/botan/"
+make -j$(nproc)
+
+# Crypto++ https://github.com/guidovranken/cryptofuzz/blob/master/docs/cryptopp.md
+cd "$MODULES_ROOT"
+git clone --depth 1 https://github.com/weidai11/cryptopp.git
+cd cryptopp/
+git rev-parse HEAD
+make libcryptopp.a -j$(nproc)
+export CXXFLAGS="$CXXFLAGS -DCRYPTOFUZZ_CRYPTOPP"
+env LIBCRYPTOPP_A_PATH `realpath libcryptopp.a`
+env CRYPTOPP_INCLUDE_PATH `realpath .`
+cd "${CRYPTOFUZZ_SRC}/modules/cryptopp/"
+make
+
+# Extract the seed corpus, docker layers are already compressed so this won't use any more space and save time when running
+cd "$FUZZ_ROOT"
+unzip cryptofuzz_data.zip
+rm cryptofuzz_data.zip
+env CRYPTOFUZZ_SEED_CORPUS `realpath cryptofuzz_seed_corpus`
+env CRYPTOFUZZ_DICT `realpath cryptofuzz-dict.txt`
+
+# Save final common flags
+env CFLAGS "$CFLAGS"
+env CXXFLAGS "$CXXFLAGS"
+env CRYPTOFUZZ_SRC "$CRYPTOFUZZ_SRC"

tests/ci/docker_images/linux-aarch/build_images.sh

@@ -12,3 +12,5 @@ docker build -t ubuntu-20.04-aarch:clang-7x ubuntu-20.04_clang-7x
 docker build -t ubuntu-20.04-aarch:clang-8x ubuntu-20.04_clang-8x
 docker build -t ubuntu-20.04-aarch:clang-9x ubuntu-20.04_clang-9x
 docker build -t ubuntu-20.04-aarch:clang-10x ubuntu-20.04_clang-10x
+# This passes in the Dockerfile in the folder but uses the parent directory for the context so it has access to cryptofuzz_data.zip
+docker build -t ubuntu-20.04-aarch:cryptofuzz -f ubuntu-20.04_cryptofuzz/Dockerfile ../

tests/ci/docker_images/linux-aarch/push_images.sh

@@ -23,3 +23,4 @@ tag_and_push_img 'ubuntu-20.04-aarch:clang-7x' "${ECS_REPO}:ubuntu-20.04_clang-7
 tag_and_push_img 'ubuntu-20.04-aarch:clang-8x' "${ECS_REPO}:ubuntu-20.04_clang-8x"
 tag_and_push_img 'ubuntu-20.04-aarch:clang-9x' "${ECS_REPO}:ubuntu-20.04_clang-9x"
 tag_and_push_img 'ubuntu-20.04-aarch:clang-10x' "${ECS_REPO}:ubuntu-20.04_clang-10x"
+tag_and_push_img 'ubuntu-20.04-aarch:cryptofuzz' "${ECS_REPO}:ubuntu-20.04_cryptofuzz"

tests/ci/docker_images/linux-aarch/ubuntu-20.04_cryptofuzz/Dockerfile

@@ -0,0 +1,25 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+FROM ubuntu-20.04-aarch:clang-10x
+
+SHELL ["/bin/bash", "-c"]
+
+RUN set -ex && \
+    apt-get update && \
+    apt-get -y --no-install-recommends upgrade && \
+    apt-get -y --no-install-recommends install \
+    make \
+    libboost-all-dev \
+    unzip && \
+    apt-get autoremove --purge -y && \
+    apt-get clean && \
+    apt-get autoclean && \
+    rm -rf /var/lib/apt/lists/* && \
+    rm -rf /tmp/*
+
+ENV FUZZ_ROOT=${DEPENDENCIES_DIR}
+ENV MODULE_ROOT="${FUZZ_ROOT}/modules"
+
+COPY build_cryptofuzz_modules.sh cryptofuzz_data.zip $FUZZ_ROOT/
+RUN set -ex && cd $FUZZ_ROOT && "./build_cryptofuzz_modules.sh"

tests/ci/docker_images/linux-x86/build_images.sh

@@ -16,6 +16,8 @@ docker build -t ubuntu-20.04:clang-7x ubuntu-20.04_clang-7x
 docker build -t ubuntu-20.04:clang-8x ubuntu-20.04_clang-8x
 docker build -t ubuntu-20.04:clang-9x ubuntu-20.04_clang-9x
 docker build -t ubuntu-20.04:clang-10x ubuntu-20.04_clang-10x
+# This passes in the Dockerfile in the folder but uses the parent directory for the context so it has access to cryptofuzz_data.zip
+docker build -t ubuntu-20.04:cryptofuzz -f ubuntu-20.04_cryptofuzz/Dockerfile ../
 docker build -t centos-7:gcc-4x centos-7_gcc-4x
 docker build -t amazonlinux-2:base amazonlinux-2_base
 docker build -t amazonlinux-2:gcc-7x amazonlinux-2_gcc-7x

tests/ci/docker_images/linux-x86/push_images.sh

@@ -22,6 +22,7 @@ tag_and_push_img 'ubuntu-20.04:clang-7x' "${ECS_REPO}:ubuntu-20.04_clang-7x"
 tag_and_push_img 'ubuntu-20.04:clang-8x' "${ECS_REPO}:ubuntu-20.04_clang-8x"
 tag_and_push_img 'ubuntu-20.04:clang-9x' "${ECS_REPO}:ubuntu-20.04_clang-9x"
 tag_and_push_img 'ubuntu-20.04:clang-10x' "${ECS_REPO}:ubuntu-20.04_clang-10x"
+tag_and_push_img 'ubuntu-20.04:cryptofuzz' "${ECS_REPO}:ubuntu-20.04_cryptofuzz"
 tag_and_push_img 'ubuntu-20.04:clang-10x_formal-verification' "${ECS_REPO}:ubuntu-20.04_clang-10x_formal-verification"
 tag_and_push_img 'ubuntu-20.04:gcc-7x' "${ECS_REPO}:ubuntu-20.04_gcc-7x"
 tag_and_push_img 'ubuntu-20.04:gcc-8x' "${ECS_REPO}:ubuntu-20.04_gcc-8x"

tests/ci/docker_images/linux-x86/ubuntu-20.04_cryptofuzz/Dockerfile

@@ -0,0 +1,25 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+FROM ubuntu-20.04:clang-10x
+
+SHELL ["/bin/bash", "-c"]
+
+RUN set -ex && \
+    apt-get update && \
+    apt-get -y --no-install-recommends upgrade && \
+    apt-get -y --no-install-recommends install \
+    make \
+    libboost-all-dev \
+    unzip && \
+    apt-get autoremove --purge -y && \
+    apt-get clean && \
+    apt-get autoclean && \
+    rm -rf /var/lib/apt/lists/* && \
+    rm -rf /tmp/*
+
+ENV FUZZ_ROOT=${DEPENDENCIES_DIR}
+ENV MODULE_ROOT="${FUZZ_ROOT}/modules"
+
+COPY build_cryptofuzz_modules.sh cryptofuzz_data.zip $FUZZ_ROOT/
+RUN set -ex && cd $FUZZ_ROOT && "./build_cryptofuzz_modules.sh"

tests/ci/run_cryptofuzz.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+set -exo pipefail
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Sourcing these files check for environment variables which may be unset so wait to enable -u
+source tests/ci/common_fuzz.sh
+source "${FUZZ_ROOT}/fuzz_env.sh"
+# After loading everything any undefined variables should fail the build
+set -u
+
+rm -rf "$BUILD_ROOT"
+mkdir -p "$BUILD_ROOT"
+cd "$BUILD_ROOT"
+
+# Build AWS-LC based on https://github.com/guidovranken/cryptofuzz/blob/master/docs/openssl.md
+cmake -DCMAKE_CXX_FLAGS="$CXXFLAGS" -DCMAKE_C_FLAGS="$CFLAGS" -DBORINGSSL_ALLOW_CXX_RUNTIME=1 \
+  -GNinja -DBUILD_TESTING=OFF -DBUILD_LIBSSL=OFF -DCMAKE_BUILD_TYPE=RelWithDebInfo ../
+ninja
+cd ../
+export CXXFLAGS="$CXXFLAGS -DCRYPTOFUZZ_BORINGSSL"
+export OPENSSL_INCLUDE_PATH=`realpath include/`
+export OPENSSL_LIBCRYPTO_A_PATH=`realpath ${BUILD_ROOT}/crypto/libcrypto.a`
+
+# Build the common OpenSSL module with AWS-LC
+cd "${CRYPTOFUZZ_SRC}/modules/openssl"
+make "-j${NUM_CPU_THREADS}"
+
+export CFLAGS="${CFLAGS} -I $OPENSSL_INCLUDE_PATH"
+export CXXFLAGS="${CXXFLAGS} -I $OPENSSL_INCLUDE_PATH"
+export LIBFUZZER_LINK="-fsanitize=fuzzer"
+
+# Build the overall cryptofuzz binary
+cd "$CRYPTOFUZZ_SRC"
+make "-j${NUM_CPU_THREADS}"
+
+# Common AWS-LC fuzzing setup, the cryptofuzz binary is in this folder so FUZZ_TEST_PATH=FUZZ_NAME
+FUZZ_NAME="cryptofuzz"
+FUZZ_TEST_PATH="$FUZZ_NAME"
+SRC_CORPUS="$CRYPTOFUZZ_SEED_CORPUS"
+
+# Perform the actual fuzzing. We want the total build time to be about an hour:
+# 4 minutes for building AWS-LC and Cryptofuzz
+# 55 minutes of fuzzing
+# 1 minutes of cleanup
+TIME_FOR_EACH_FUZZ=3300
+
+# Some fuzz tests can take a while but still pass. This is a tradeoff: less false positive noise, but some inputs that take
+# a long time could lead to a denial of service avenue. We're mostly interested in correctness and memory safety at this
+# time so we're willing to take the fit on fuzz speed
+FUZZ_TEST_TIMEOUT=30
+
+# Call the common fuzzing logic
+run_fuzz_test