diff --git a/infra/base-images/base-builder/compile b/infra/base-images/base-builder/compile index d0f45bb73fef..1c10d9e234e7 100755 --- a/infra/base-images/base-builder/compile +++ b/infra/base-images/base-builder/compile @@ -21,6 +21,9 @@ sysctl -w vm.mmap_rnd_bits=28 OSS_FUZZ_ON_DEMAND="${OSS_FUZZ_ON_DEMAND:-0}" +# Used for Rust introspector builds +RUST_SANITIZER=$SANITIZER + if [ "$FUZZING_LANGUAGE" = "jvm" ]; then if [ "$FUZZING_ENGINE" != "libfuzzer" ] && [ "$FUZZING_ENGINE" != "wycheproof" ]; then echo "ERROR: JVM projects can be fuzzed with libFuzzer or tested with wycheproof engines only." @@ -36,6 +39,15 @@ if [ "$FUZZING_LANGUAGE" = "jvm" ]; then fi fi +if [ "$FUZZING_LANGUAGE" = "rust" ]; then + if [ "$SANITIZER" = "introspector" ]; then + # introspector sanitizer flag will cause cargo build to fail. Rremove it + # temporarily, RUST_SANITIZER will hold the original sanitizer. + export SANITIZER=address + fi +fi + + if [ "$FUZZING_LANGUAGE" = "javascript" ]; then if [ "$FUZZING_ENGINE" != "libfuzzer" ]; then echo "ERROR: JavaScript projects can be fuzzed with libFuzzer engine only." @@ -111,7 +123,9 @@ fi # use RUSTFLAGS. # FIXME: Support code coverage once support is in. # See https://github.com/rust-lang/rust/issues/34701. -if [ "$SANITIZER" != "undefined" ] && [ "$SANITIZER" != "coverage" ] && [ "$SANITIZER" != "none" ] && [ "$ARCHITECTURE" != 'i386' ]; then +if [ "$RUST_SANITIZER" == "introspector" ]; then + export RUSTFLAGS="-Cdebuginfo=2 -Cforce-frame-pointers" +elif [ "$SANITIZER" != "undefined" ] && [ "$SANITIZER" != "coverage" ] && [ "$SANITIZER" != "none" ] && [ "$ARCHITECTURE" != 'i386' ]; then export RUSTFLAGS="--cfg fuzzing -Zsanitizer=${SANITIZER} -Cdebuginfo=1 -Cforce-frame-pointers" else export RUSTFLAGS="--cfg fuzzing -Cdebuginfo=1 -Cforce-frame-pointers" @@ -188,7 +202,7 @@ EOF export CXXFLAGS="$CXXFLAGS -fno-sanitize=leak" fi -if [ "$SANITIZER" = "introspector" ]; then +if [ "$SANITIZER" = "introspector" ] || [ "$RUST_SANITIZER" = "introspector" ]; then export AR=llvm-ar export NM=llvm-nm export RANLIB=llvm-ranlib @@ -210,10 +224,24 @@ if [ "$SANITIZER" = "introspector" ]; then apt-get install -y libjpeg-dev zlib1g-dev libyaml-dev python3 -m pip install --upgrade pip setuptools - python3 -m pip install cxxfilt pyyaml beautifulsoup4 lxml soupsieve + python3 -m pip install cxxfilt pyyaml beautifulsoup4 lxml soupsieve rust-demangler python3 -m pip install --prefer-binary matplotlib - python3 /fuzz-introspector/src/main.py light + # Install Fuzz-Introspector + pushd /fuzz-introspector/src + python3 -m pip install . + popd + + if [ "$FUZZING_LANGUAGE" = "python" ]; then + python3 /fuzz-introspector/src/main.py light --language=python + elif [ "$FUZZING_LANGUAGE" = "jvm" ]; then + python3 /fuzz-introspector/src/main.py light --language=jvm + elif [ "$FUZZING_LANGUAGE" = "rust" ]; then + python3 /fuzz-introspector/src/main.py light --language=rust + else + python3 /fuzz-introspector/src/main.py light + fi + rsync -avu --delete "$SRC/inspector/" "$OUT/inspector" fi @@ -280,7 +308,7 @@ else fi fi -if [ "$SANITIZER" = "introspector" ]; then +if [ "$SANITIZER" = "introspector" ] || [ "$RUST_SANITIZER" = "introspector" ]; then unset CXXFLAGS unset CFLAGS export G_ANALYTICS_TAG="G-8WTFM1Y62J" @@ -295,6 +323,21 @@ if [ "$SANITIZER" = "introspector" ]; then mkdir -p $SRC/my-fi-data find $OUT/ -name *.data -exec mv {} $SRC/my-fi-data/ \; find $OUT/ -name *.data.yaml -exec mv {} $SRC/my-fi-data/ \; + elif [ "$FUZZING_LANGUAGE" = "rust" ]; then + echo "GOING rust route" + + # Run the rust frontend + pushd /fuzz-introspector/frontends/rust/rust_function_analyser + cargo run -- $SRC + + # Move files temporarily to fix workflow of other languages. + mkdir -p $SRC/my-fi-data + find ./ -name "*.data" -exec mv {} $SRC/my-fi-data/ \; + find ./ -name "*.data.yaml" -exec mv {} $SRC/my-fi-data/ \; + popd + + # Restore the sanitizer flag for rust + export SANITIZER="introspector" fi mkdir -p $SRC/inspector @@ -335,6 +378,12 @@ if [ "$SANITIZER" = "introspector" ]; then REPORT_ARGS="$REPORT_ARGS --language=jvm" python3 /fuzz-introspector/src/main.py report $REPORT_ARGS rsync -avu --delete "$SRC/inspector/" "$OUT/inspector" + elif [ "$FUZZING_LANGUAGE" = "rust" ]; then + echo "GOING rust route" + REPORT_ARGS="$REPORT_ARGS --target_dir=$SRC/inspector" + REPORT_ARGS="$REPORT_ARGS --language=rust" + python3 /fuzz-introspector/src/main.py report $REPORT_ARGS + rsync -avu --delete "$SRC/inspector/" "$OUT/inspector" else # C/C++ diff --git a/infra/base-images/base-clang/Dockerfile b/infra/base-images/base-clang/Dockerfile index 8f6486896dcf..d403bb07d876 100644 --- a/infra/base-images/base-clang/Dockerfile +++ b/infra/base-images/base-clang/Dockerfile @@ -36,7 +36,7 @@ RUN apt-get update && apt-get install -y wget sudo && \ RUN apt-get update && apt-get install -y git && \ git clone https://github.com/ossf/fuzz-introspector.git fuzz-introspector && \ cd fuzz-introspector && \ - git checkout 5924aea8bcfe1fbdac9dc815adff91d3ee51f52b && \ + git checkout 74917384c5a4e368d900862b4bd3d16ce3fe5dd8 && \ git submodule init && \ git submodule update && \ apt-get autoremove --purge -y git && \ diff --git a/infra/build/functions/build_and_run_coverage.py b/infra/build/functions/build_and_run_coverage.py index 48ba786e8b1c..9ee40fd58a3e 100755 --- a/infra/build/functions/build_and_run_coverage.py +++ b/infra/build/functions/build_and_run_coverage.py @@ -46,7 +46,7 @@ 'c', 'c++', 'go', 'jvm', 'rust', 'swift', 'python' ] -LANGUAGES_WITH_INTROSPECTOR_SUPPORT = ['c', 'c++', 'python', 'jvm'] +LANGUAGES_WITH_INTROSPECTOR_SUPPORT = ['c', 'c++', 'python', 'jvm', 'rust'] class Bucket: # pylint: disable=too-few-public-methods diff --git a/infra/build/functions/build_lib.py b/infra/build/functions/build_lib.py index beb937c947bc..f244a766e97e 100644 --- a/infra/build/functions/build_lib.py +++ b/infra/build/functions/build_lib.py @@ -275,11 +275,13 @@ def download_coverage_data_steps(project_name, latest, bucket_name, out_dir): bucket_url = f'gs://{bucket_name}/{project_name}/textcov_reports/{latest}/*' steps.append({ 'name': 'gcr.io/cloud-builders/gsutil', - 'args': ['-m', 'cp', '-r', bucket_url, coverage_data_path] + 'args': ['-m', 'cp', '-r', bucket_url, coverage_data_path], + 'allowFailure': True }) steps.append({ 'name': 'gcr.io/oss-fuzz-base/base-runner', - 'args': ['bash', '-c', f'ls -lrt {out_dir}/textcov_reports'] + 'args': ['bash', '-c', f'ls -lrt {out_dir}/textcov_reports'], + 'allowFailure': True }) return steps @@ -415,6 +417,7 @@ def get_docker_build_step(image_names, 'name': DOCKER_TOOL_IMAGE, 'args': args, 'dir': directory, + 'id': f'build-{architecture}', } # Handle buildkit args # Note that we mutate "args" after making it a value in step. @@ -460,10 +463,11 @@ def get_project_image_steps( # pylint: disable=too-many-arguments steps.extend(get_pull_test_images_steps(config.test_image_suffix)) src_root = 'oss-fuzz' if not experiment else '.' - docker_build_step = get_docker_build_step([image], - os.path.join('projects', name), - src_root=src_root, - cache_image=cache_image) + docker_build_step = get_docker_build_step( + [image, _get_unsafe_name(name)], + os.path.join('projects', name), + src_root=src_root, + cache_image=cache_image) steps.append(docker_build_step) if srcmap: srcmap_step_id = get_srcmap_step_id() @@ -475,7 +479,7 @@ def get_project_image_steps( # pylint: disable=too-many-arguments ], 'env': [ 'OSSFUZZ_REVISION=$REVISION_ID', - 'FUZZING_LANGUAGE=%s' % language, + f'FUZZING_LANGUAGE={language}', ], 'id': srcmap_step_id }]) @@ -496,15 +500,31 @@ def get_project_image_steps( # pylint: disable=too-many-arguments 'args': ['buildx', 'use', builder_name] }, ]) - docker_build_arm_step = get_docker_build_step([image], - os.path.join( - 'projects', name), - architecture=_ARM64) + docker_build_arm_step = get_docker_build_step( + [image, _get_unsafe_name(name)], + os.path.join('projects', name), + architecture=_ARM64) steps.append(docker_build_arm_step) + if (not experiment and not config.testing and + config.build_type == 'fuzzing' and language in ('c', 'c++')): + # Push so that historical bugs are reproducible. + push_step = { + 'name': 'gcr.io/cloud-builders/docker', + 'args': ['push', _get_unsafe_name(name)], + 'id': 'push-image', + 'waitFor': [docker_build_step['id']], + 'allowFailure': True + } + steps.append(push_step) + return steps +def _get_unsafe_name(name): + return f'us-central1-docker.pkg.dev/oss-fuzz/unsafe/{name}' + + def get_logs_url(build_id): """Returns url that displays the build logs.""" return ( diff --git a/infra/build/functions/build_project.py b/infra/build/functions/build_project.py index b8eca8eb0190..2a6c4ad9b156 100755 --- a/infra/build/functions/build_project.py +++ b/infra/build/functions/build_project.py @@ -19,8 +19,6 @@ Usage: build_project.py """ -from __future__ import print_function - import argparse from dataclasses import dataclass import datetime @@ -77,6 +75,7 @@ class Config: experiment: bool = False # TODO(ochang): This should be different per engine+sanitizer combination. upload_build_logs: str = None + build_type: str = None WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)') @@ -476,7 +475,6 @@ def get_build_steps_for_project(project, upload_steps = get_upload_steps(project, build, timestamp, config.testing) build_steps.extend(upload_steps) - return build_steps @@ -629,7 +627,7 @@ def get_args(description): return parser.parse_args() -def create_config_from_commandline(args): +def create_config(args, build_type): """Create a Config object from parsed command line |args|.""" upload = not args.experiment return Config(testing=args.testing, @@ -637,7 +635,8 @@ def create_config_from_commandline(args): branch=args.branch, parallel=args.parallel, upload=upload, - experiment=args.experiment) + experiment=args.experiment, + build_type=build_type) def build_script_main(script_description, get_build_steps_func, build_type): @@ -650,7 +649,7 @@ def build_script_main(script_description, get_build_steps_func, build_type): credentials = oauth2client.client.GoogleCredentials.get_application_default() error = False - config = create_config_from_commandline(args) + config = create_config(args, build_type) for project_name in args.projects: logging.info('Getting steps for: "%s".', project_name) try: diff --git a/infra/build/functions/test_data/expected_build_steps.json b/infra/build/functions/test_data/expected_build_steps.json index 065c76c7bbbf..a35cb9a6ce80 100644 --- a/infra/build/functions/test_data/expected_build_steps.json +++ b/infra/build/functions/test_data/expected_build_steps.json @@ -14,9 +14,12 @@ "build", "--tag", "gcr.io/oss-fuzz/test-project", + "--tag", + "us-central1-docker.pkg.dev/oss-fuzz/unsafe/test-project", "." ], - "dir": "oss-fuzz/projects/test-project" + "dir": "oss-fuzz/projects/test-project", + "id": "build-x86_64" }, { "name": "gcr.io/oss-fuzz/test-project", @@ -68,9 +71,12 @@ "--load", "--tag", "gcr.io/oss-fuzz/test-project-aarch64", + "--tag", + "us-central1-docker.pkg.dev/oss-fuzz/unsafe/test-project-aarch64", "." ], - "dir": "oss-fuzz/projects/test-project" + "dir": "oss-fuzz/projects/test-project", + "id": "build-aarch64" }, { "name": "gcr.io/cloud-builders/docker", diff --git a/infra/build/functions/test_data/expected_centipede_build_steps.json b/infra/build/functions/test_data/expected_centipede_build_steps.json index db92d885ec2d..885c803d312a 100644 --- a/infra/build/functions/test_data/expected_centipede_build_steps.json +++ b/infra/build/functions/test_data/expected_centipede_build_steps.json @@ -13,10 +13,13 @@ "args": [ "build", "--tag", - "gcr.io/oss-fuzz/test-project", + "gcr.io/oss-fuzz/test-project", + "--tag", + "us-central1-docker.pkg.dev/oss-fuzz/unsafe/test-project", "." ], - "dir": "oss-fuzz/projects/test-project" + "dir": "oss-fuzz/projects/test-project", + "id": "build-x86_64" }, { "name": "gcr.io/oss-fuzz/test-project", diff --git a/infra/build/functions/test_data/expected_coverage_build_steps.json b/infra/build/functions/test_data/expected_coverage_build_steps.json index 9fce94c44bf2..493c6ce25d2a 100644 --- a/infra/build/functions/test_data/expected_coverage_build_steps.json +++ b/infra/build/functions/test_data/expected_coverage_build_steps.json @@ -14,9 +14,12 @@ "build", "--tag", "gcr.io/oss-fuzz/test-project", + "--tag", + "us-central1-docker.pkg.dev/oss-fuzz/unsafe/test-project", "." ], - "dir": "oss-fuzz/projects/test-project" + "dir": "oss-fuzz/projects/test-project", + "id": "build-x86_64" }, { "name": "gcr.io/oss-fuzz/test-project", diff --git a/infra/build/functions/test_data/expected_trial_build_steps.json b/infra/build/functions/test_data/expected_trial_build_steps.json index 1497f7e05a7a..a41da778516c 100644 --- a/infra/build/functions/test_data/expected_trial_build_steps.json +++ b/infra/build/functions/test_data/expected_trial_build_steps.json @@ -167,9 +167,12 @@ "build", "--tag", "gcr.io/oss-fuzz/skcms", + "--tag", + "us-central1-docker.pkg.dev/oss-fuzz/unsafe/skcms", "." ], - "dir": "oss-fuzz/projects/skcms" + "dir": "oss-fuzz/projects/skcms", + "id": "build-x86_64" }, { "name": "gcr.io/oss-fuzz/skcms", diff --git a/infra/experimental/chronos/README.md b/infra/experimental/chronos/README.md index 9f6a0e1846d4..4dfe1baeed8a 100644 --- a/infra/experimental/chronos/README.md +++ b/infra/experimental/chronos/README.md @@ -1,23 +1,20 @@ -# Usage -Under `OSS-Fuzz` root directory: -```bash -export PROJECT=libiec61850 -export FUZZ_TARGET=fuzz_mms_decode.c -export FUZZING_LANGUAGE=c +# Chronos: rebuilding OSS-Fuzz harnesses using cached builds -infra/experimental/chronos/prepare-recompile "$PROJECT" "$FUZZ_TARGET" "$FUZZING_LANGUAGE" -python infra/helper.py build_image "$PROJECT" -# AddressSanitizer. -docker run -ti --entrypoint="/bin/sh" --env SANITIZER="address" --name "${PROJECT}-origin-asan" "gcr.io/oss-fuzz/${PROJECT}" -c "compile && rm -rf /out/*" -docker commit --change 'CMD ["compile"] --change 'ENTRYPOINT /bin/sh' "${PROJECT}-origin-asan" "gcr.io/oss-fuzz/${PROJECT}-ofg-cached-asan" -docker run -ti --entrypoint="recompile" "gcr.io/oss-fuzz/${PROJECT}-ofg-cached-asan" +## Usage locally -# Coverage measurement. -docker run -ti --entrypoint="/bin/sh" --env SANITIZER="coverage" --name "${PROJECT}-origin-cov" "gcr.io/oss-fuzz/${PROJECT}" -c "compile && rm -rf /out/*" -docker commit --change 'CMD ["compile"] --change 'ENTRYPOINT /bin/sh' "${PROJECT}-origin-cov" "gcr.io/oss-fuzz/${PROJECT}-ofg-cached-cov" -docker run -ti --entrypoint="recompile" "gcr.io/oss-fuzz/${PROJECT}-ofg-cached-cov" -``` +**Example 1: htslib** + +From the OSS-Fuzz root -# Assumptions -1. Fuzzer: Chronos assumes `libFuzzer`. Other fuzzers are not well-supported, but may work by setting ENV `FUZZING_ENGINE` in project's `Dockerfile`. -2. Sanitizer: Chronos assumes `AddressSanitizer`. Other sanitizers may work by adding setting ENV `SANITIZER` in project's `Dockerfile`. +```sh +$ RUN_ALL=1 ./infra/experimental/chronos/build_cache_local.sh htslib c +... +... +Vanilla compile time: +17 +Replay worked +Replay compile time: +2 +Ccache compile time: +9 +``` diff --git a/infra/experimental/chronos/build_cache_local.sh b/infra/experimental/chronos/build_cache_local.sh index 3a7e60a9930e..df2edba312cc 100755 --- a/infra/experimental/chronos/build_cache_local.sh +++ b/infra/experimental/chronos/build_cache_local.sh @@ -20,32 +20,100 @@ _FUZZING_LANGUAGE=$2 BASE=$PWD +# Step 1: build the base image cd projects/${_PROJECT} docker build -t gcr.io/oss-fuzz/${_PROJECT} . -mkdir -p ccaches/${_PROJECT} + +# Step 2: create a container where `compile` has run which enables ccaching +# and also generates a replay build script. cd ${BASE} +mkdir -p ccaches/${_PROJECT} +mkdir -p build/out/${_PROJECT} B_START=$SECONDS docker run \ --entrypoint=/bin/bash \ --env=SANITIZER=address \ --env=CCACHE_DIR=/workspace/ccache \ --env=FUZZING_LANGUAGE=${_FUZZING_LANGUAGE} \ + --env=CAPTURE_REPLAY_SCRIPT=1 \ --name=${_PROJECT}-origin-asan \ -v=$PWD/ccaches/${_PROJECT}/ccache:/workspace/ccache \ + -v=$PWD/build/out/${_PROJECT}/:/out/ \ gcr.io/oss-fuzz/${_PROJECT} \ -c \ "export PATH=/ccache/bin:\$PATH && compile" B_TIME=$(($SECONDS - $B_START)) -# Prepare Dockerfile for ccache +# Step 3: save (commit, locally) the cached container as an image +docker container commit ${_PROJECT}-origin-asan local/ossfuzz/${_PROJECT}-origin-asan + + +# Step 4: save the list of executables created from a vanilla build. This is +# needed for validating if replay and ccaching works. +# notes: run a shell the container with e.g. +# `docker run --entrypoint /bin/bash -it local/ossfuzz/htslib-origin-asan` +executables_vanilla="$(find ./build/out/${_PROJECT} -executable -type f | sort)" + + +# Step 5: Build with replay enabled, and validate the executables are the same +# in terms of naming. +# Note that an important step is removing everything in $OUT/ which is done +# in the docker command. +R_START=$SECONDS +docker run \ + --entrypoint=/bin/bash \ + --env=SANITIZER=address \ + --env=REPLAY_ENABLED=1 \ + --env=FUZZING_LANGUAGE=${_FUZZING_LANGUAGE} \ + -v=$PWD/build/out/${_PROJECT}/:/out/ \ + --name=${_PROJECT}-origin-asan-replay-recached \ + local/ossfuzz/${_PROJECT}-origin-asan \ + -c \ + "export PATH=/ccache/bin:\$PATH && rm -rf /out/* && compile" +R_TIME=$(($SECONDS - $R_START)) + +# Step 6: Extract the newly build executables +executables_replay="$(find ./build/out/${_PROJECT}/ -executable -type f | sort)" + +echo "Executables vanilla: " +echo ${executables_vanilla} + +echo "------------------------------------------------------" +echo "Executables replay: " +echo ${executables_replay} + +# Step 7: match executables from vanilla builds and replay builds. +# If this step is successful, then the process can exit as it's ready. +if [[ "$executables_replay" == "$executables_vanilla" ]] +then + echo "Replay worked" + echo "Vanilla compile time:" + echo ${B_TIME} + echo "Replay compile time:" + echo ${R_TIME} + + if [ -n "${RUN_ALL+1}" ]; then + exit 0 + fi +else + echo "Replay did not work" +fi + +# Step 8: prepare Dockerfile for ccache cp -rf ccaches/${_PROJECT}/ccache ./projects/${_PROJECT}/ccache-cache infra/experimental/chronos/prepare-ccache ${_PROJECT} cd projects/${_PROJECT} + +# Step 9: Build an image with CCache's new items (modifications are done on the +# dockerfile) docker build -t us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-address . +cd ${BASE} + +# Step 10: Run a `compile` with ccache's image. # Run the ccache build A_START=$SECONDS docker run \ @@ -53,13 +121,34 @@ docker run \ --env=SANITIZER=address \ --env=FUZZING_LANGUAGE=${_FUZZING_LANGUAGE} \ --name=${_PROJECT}-origin-asan-recached \ + -v=$PWD/build/out/${_PROJECT}/:/out/ \ us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-address \ -c \ - "export PATH=/ccache/bin:\$PATH && compile" + "export PATH=/ccache/bin:\$PATH && rm -rf /out/* && compile" A_TIME=$(($SECONDS - $A_START)) -echo "No cache: " -echo ${B_TIME} +# Step 11: extract the executables from the ccache build +executables_ccache="$(find ./build/out/${_PROJECT}/ -executable -type f | sort)" + + +# Step 12: validate the ccache builds are successful +if [[ "$executables_ccache" == "$executables_vanilla" ]] +then + echo "Vanilla compile time:" + echo ${B_TIME} + if [[ "$executables_replay" == "$executables_vanilla" ]] + then + echo "Replay worked" + echo "Replay compile time:" + echo ${R_TIME} + fi + + echo "Ccache compile time: " + echo ${A_TIME} + + exit 0 +else + echo "Replay and ccaching did not work." +fi + -echo "After cache: " -echo ${A_TIME} diff --git a/infra/experimental/chronos/cloudbuild.yaml b/infra/experimental/chronos/cloudbuild.yaml index f97c2567868b..2e4ae186b121 100644 --- a/infra/experimental/chronos/cloudbuild.yaml +++ b/infra/experimental/chronos/cloudbuild.yaml @@ -109,7 +109,7 @@ steps: images: - us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-address - us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-coverage -timeout: 1800s +timeout: 72000s # 20 hours, same as build_lib.py logsBucket: oss-fuzz-gcb-logs tags: - ${_PROJECT} diff --git a/infra/experimental/chronos/prepare-ccache b/infra/experimental/chronos/prepare-ccache index e44e1d7cac4b..3b0f56719680 100755 --- a/infra/experimental/chronos/prepare-ccache +++ b/infra/experimental/chronos/prepare-ccache @@ -18,5 +18,5 @@ PROJECT=$1 { echo "COPY ccache-cache/ /ccache/cache"; - echo "ENV PATH=/ccache/bin:$PATH" + echo "ENV PATH=\"/ccache/bin:\$PATH\"" } >> "projects/$PROJECT/Dockerfile" diff --git a/projects/abseil-cpp/build.sh b/projects/abseil-cpp/build.sh index 3fd7d194d969..3896f0482787 100644 --- a/projects/abseil-cpp/build.sh +++ b/projects/abseil-cpp/build.sh @@ -14,6 +14,7 @@ # ################################################################################ +export USE_BAZEL_VERSION=7.4.0 # Disable `layering_check` feature. # As per https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=63223, it breaks # the build. Someone could figure out exactly why it breaks the build, but just diff --git a/projects/boost/boost_regex_pattern_fuzzer.cc b/projects/boost/boost_regex_pattern_fuzzer.cc index a9341b54c8df..d742101d4fc6 100644 --- a/projects/boost/boost_regex_pattern_fuzzer.cc +++ b/projects/boost/boost_regex_pattern_fuzzer.cc @@ -23,7 +23,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { // Currently, we just consume all the fuzzed corpus into the regex pattern std::string regex_string = fdp.ConsumeRemainingBytesAsString(); const uint8_t where_array[] = {0x48,0x48,0x48,0x48,0x48,0x48,0x48,0x48,0x48,0x48,0x48,0x48,0x48,0x48,0x48}; - std::string random(where_array, where_array + sizeof(where_array)); + std::string random(where_array, where_array + sizeof(where_array)); std::string empty(""); std::string spaces(" "); try { @@ -38,8 +38,17 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { #endif for (const auto& where : wheres) { - boost::match_results what; - bool match = boost::regex_match(where, what, e, boost::match_default | boost::match_partial | boost::match_perl | boost::match_posix | boost::match_any); + try { + boost::match_results what; + bool match = boost::regex_match(where, what, e, boost::match_default | boost::match_partial | boost::match_posix | boost::match_any); + } catch(...) { + } + + try { + boost::match_results what; + bool match = boost::regex_match(where, what, e, boost::match_default | boost::match_partial | boost::match_perl | boost::match_any); + } catch(...) { + } } } catch(...) { } diff --git a/projects/cri-o/Dockerfile b/projects/cri-o/Dockerfile index 4f062be25fef..67f2d0f302c5 100644 --- a/projects/cri-o/Dockerfile +++ b/projects/cri-o/Dockerfile @@ -14,11 +14,17 @@ # ################################################################################ +FROM gcr.io/oss-fuzz-base/base-builder FROM gcr.io/oss-fuzz-base/base-builder-go RUN apt-get update && apt-get install -y libaio-dev autoconf gettext texinfo \ libbtrfs-dev git libassuan-dev libdevmapper-dev libglib2.0-dev libc6-dev \ libgpgme-dev libgpg-error-dev libseccomp-dev libsystemd-dev libselinux1-dev \ pkg-config go-md2man libudev-dev software-properties-common systemd +RUN wget https://go.dev/dl/go1.23.4.linux-amd64.tar.gz \ + && mkdir temp-go \ + && rm -rf /root/.go/* \ + && tar -C temp-go/ -xzf go1.23.4.linux-amd64.tar.gz \ + && mv temp-go/go/* /root/.go/ RUN git clone --depth 1 https://github.com/cri-o/cri-o RUN git clone --depth 1 https://github.com/cncf/cncf-fuzzing COPY build.sh $SRC/ diff --git a/projects/cxxopts/build.sh b/projects/cxxopts/build.sh index 577daee07d98..dd264f367e4c 100755 --- a/projects/cxxopts/build.sh +++ b/projects/cxxopts/build.sh @@ -15,4 +15,5 @@ # ################################################################################ +export USE_BAZEL_VERSION=7.4.0 bazel_build_fuzz_tests diff --git a/projects/dgraph/build.sh b/projects/dgraph/build.sh index 184a2217617c..6bdec5f719d5 100644 --- a/projects/dgraph/build.sh +++ b/projects/dgraph/build.sh @@ -18,4 +18,4 @@ mv $SRC/fuzz_parser_test.go $SRC/dgraph/dql/ printf "package dql\nimport _ \"github.com/AdamKorcz/go-118-fuzz-build/testing\"\n" > dql/register.go go mod tidy -compile_native_go_fuzzer github.com/dgraph-io/dgraph/dql FuzzParserTest parser_fuzzer +compile_native_go_fuzzer github.com/dgraph-io/dgraph/v24/dql FuzzParserTest parser_fuzzer diff --git a/projects/dgraph/project.yaml b/projects/dgraph/project.yaml index 72199ee5745e..04796f4a2783 100644 --- a/projects/dgraph/project.yaml +++ b/projects/dgraph/project.yaml @@ -1,9 +1,10 @@ homepage: "https://dgraph.io" main_repo: "https://github.com/dgraph-io/dgraph" -primary_contact: "security@dgraph.io" +primary_contact: "security@hypermode.com" auto_ccs : - - "harshil@dgraph.io" - - "ryan@dgraph.io" + - "harshil@hypermode.com" + - "ryan@hypermode.com" + - "aman@hypermode.com" language: go fuzzing_engines: - libfuzzer diff --git a/projects/distribution/Dockerfile b/projects/distribution/Dockerfile index 1c9170301e43..f4317ceb6e00 100644 --- a/projects/distribution/Dockerfile +++ b/projects/distribution/Dockerfile @@ -17,5 +17,6 @@ FROM gcr.io/oss-fuzz-base/base-builder-go RUN git clone --depth 1 https://github.com/distribution/distribution RUN git clone --depth 1 https://github.com/cncf/cncf-fuzzing +RUN git clone --depth 1 https://github.com/AdamKorcz/go-118-fuzz-build --branch=november-backup COPY build.sh $SRC/ WORKDIR $SRC/distribution diff --git a/projects/etcd/Dockerfile b/projects/etcd/Dockerfile index 88093fcfc3e5..c69c26832bc3 100644 --- a/projects/etcd/Dockerfile +++ b/projects/etcd/Dockerfile @@ -18,5 +18,10 @@ FROM gcr.io/oss-fuzz-base/base-builder-go RUN git clone --depth 1 https://github.com/etcd-io/etcd RUN git clone --depth 1 https://github.com/etcd-io/raft RUN git clone --depth 1 https://github.com/cncf/cncf-fuzzing +RUN wget https://go.dev/dl/go1.23.4.linux-amd64.tar.gz \ + && mkdir temp-go \ + && rm -rf /root/.go/* \ + && tar -C temp-go/ -xzf go1.23.4.linux-amd64.tar.gz \ + && mv temp-go/go/* /root/.go/ COPY build.sh $SRC/ WORKDIR $SRC/etcd diff --git a/projects/faad2/build.sh b/projects/faad2/build.sh index 063ef6dc1376..dd264f367e4c 100644 --- a/projects/faad2/build.sh +++ b/projects/faad2/build.sh @@ -14,4 +14,6 @@ # limitations under the License. # ################################################################################ + +export USE_BAZEL_VERSION=7.4.0 bazel_build_fuzz_tests diff --git a/projects/istio/Dockerfile b/projects/istio/Dockerfile index e9ddcdd95946..73137fc7222a 100644 --- a/projects/istio/Dockerfile +++ b/projects/istio/Dockerfile @@ -16,6 +16,11 @@ # Setup the builder for Istio. The standard Go builder is sufficient. FROM gcr.io/oss-fuzz-base/base-builder-go +RUN wget https://go.dev/dl/go1.23.4.linux-amd64.tar.gz \ + && mkdir temp-go \ + && rm -rf /root/.go/* \ + && tar -C temp-go/ -xzf go1.23.4.linux-amd64.tar.gz \ + && mv temp-go/go/* /root/.go/ RUN git clone --depth 1 https://github.com/istio/istio COPY build.sh $SRC/ WORKDIR $SRC/istio diff --git a/projects/kimageformats/Dockerfile b/projects/kimageformats/Dockerfile index 1e486ca00245..72aa8d01d114 100644 --- a/projects/kimageformats/Dockerfile +++ b/projects/kimageformats/Dockerfile @@ -26,12 +26,12 @@ RUN git clone --depth 1 -b master https://invent.kde.org/frameworks/extra-cmake- RUN git clone --depth 1 --branch=dev git://code.qt.io/qt/qtbase.git RUN git clone --depth 1 -b master https://invent.kde.org/frameworks/karchive.git RUN git clone --depth 1 -b master https://invent.kde.org/frameworks/kimageformats.git -RUN git clone --depth 1 -b v3.9.1 https://aomedia.googlesource.com/aom +RUN git clone --depth 1 -b v3.11.0 https://aomedia.googlesource.com/aom RUN git clone --depth 1 -b v1.1.1 https://github.com/AOMediaCodec/libavif.git RUN git clone --depth 1 https://github.com/strukturag/libde265.git -RUN git clone --depth 1 -b v2.5.2 https://github.com/uclouvain/openjpeg.git +RUN git clone --depth 1 -b v2.5.3 https://github.com/uclouvain/openjpeg.git RUN git clone --depth 1 https://github.com/strukturag/libheif.git -RUN git clone --depth=1 --branch v0.10.x --recursive --shallow-submodules https://github.com/libjxl/libjxl.git +RUN git clone --depth=1 --branch v0.11.x --recursive --shallow-submodules https://github.com/libjxl/libjxl.git RUN git clone --depth 1 https://github.com/LibRaw/LibRaw RUN git clone --depth 1 https://github.com/mircomir/jxrlib.git COPY build.sh $SRC diff --git a/projects/kimageformats/build.sh b/projects/kimageformats/build.sh index dfbf942ae6c6..f9bec5ae4fe4 100644 --- a/projects/kimageformats/build.sh +++ b/projects/kimageformats/build.sh @@ -152,7 +152,7 @@ sed -i "s/static const int MAX_IMAGE_WIDTH = 32768;/static const int MAX_IMAGE_W sed -i "s/static const int MAX_IMAGE_HEIGHT = 32768;/static const int MAX_IMAGE_HEIGHT = 8192;/g" libheif/security_limits.h mkdir build cd build -cmake -DBUILD_SHARED_LIBS=OFF -DENABLE_PLUGIN_LOADING=OFF -DWITH_DAV1D=OFF -DWITH_EXAMPLES=OFF -DWITH_LIBDE265=ON -DWITH_RAV1E=OFF -DWITH_RAV1E_PLUGIN=OFF -DWITH_SvtEnc=OFF -DWITH_SvtEnc_PLUGIN=OFF -DWITH_X265=OFF -DWITH_OpenJPEG_DECODER=ON .. +cmake -DBUILD_SHARED_LIBS=OFF -DENABLE_PLUGIN_LOADING=OFF -DWITH_DAV1D=OFF -DWITH_EXAMPLES=OFF -DWITH_LIBDE265=ON -DWITH_RAV1E=OFF -DWITH_RAV1E_PLUGIN=OFF -DWITH_SvtEnc=OFF -DWITH_SvtEnc_PLUGIN=OFF -DWITH_X265=OFF -DWITH_OpenJPEG_DECODER=ON -DWITH_OpenH264_DECODER=OFF .. make -j$(nproc) make install -j$(nproc)