From d7d31c01c76a8951c33b3ea41e644bbbe7a0cff1 Mon Sep 17 00:00:00 2001 From: DavidKorczynski Date: Fri, 13 Dec 2024 22:01:47 +0000 Subject: [PATCH 01/17] chronos: fix PATH setting (#12845) Currently the $PATH bit is expanded, i.e. the ENV string becomes the PATH of the host system, as opposed to the docker container's PATH. We should escape the `$` to ensure we don't overwrite the full PATH --- infra/experimental/chronos/prepare-ccache | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/experimental/chronos/prepare-ccache b/infra/experimental/chronos/prepare-ccache index e44e1d7cac4b..3b0f56719680 100755 --- a/infra/experimental/chronos/prepare-ccache +++ b/infra/experimental/chronos/prepare-ccache @@ -18,5 +18,5 @@ PROJECT=$1 { echo "COPY ccache-cache/ /ccache/cache"; - echo "ENV PATH=/ccache/bin:$PATH" + echo "ENV PATH=\"/ccache/bin:\$PATH\"" } >> "projects/$PROJECT/Dockerfile" From ecfd46245cb0840d0dbd149c6fb83af0a1b5553a Mon Sep 17 00:00:00 2001 From: Christian Mazakas <5543573+cmazakas@users.noreply.github.com> Date: Sat, 14 Dec 2024 01:25:33 -0800 Subject: [PATCH 02/17] update boost regex test for new behavior (#12844) In future versions of Boost, this fuzzing test will throw unconditionally because it's erroneously mixing two different matching modes (`match_perl | match_posix`). Right now, the test is fine but eventually, it will be broken and no longer function as it should. You can see the new behavior being tested on the develop branch here: https://github.com/boostorg/regex/blob/0cbaa4ef1740c88106337cf0ba5bf5c73b62863c/test/issue232.cpp#L58 By splitting the regex calls here into their own try-catch blocks, we also ensure a more robust test of the regex matching code by ensuring that both modes are always called always. The other tests seemed unaffected by this change. @jzmaddock might have some opinions on this too, as he's the primary Regex author. --- projects/boost/boost_regex_pattern_fuzzer.cc | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/projects/boost/boost_regex_pattern_fuzzer.cc b/projects/boost/boost_regex_pattern_fuzzer.cc index a9341b54c8df..d742101d4fc6 100644 --- a/projects/boost/boost_regex_pattern_fuzzer.cc +++ b/projects/boost/boost_regex_pattern_fuzzer.cc @@ -23,7 +23,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { // Currently, we just consume all the fuzzed corpus into the regex pattern std::string regex_string = fdp.ConsumeRemainingBytesAsString(); const uint8_t where_array[] = {0x48,0x48,0x48,0x48,0x48,0x48,0x48,0x48,0x48,0x48,0x48,0x48,0x48,0x48,0x48}; - std::string random(where_array, where_array + sizeof(where_array)); + std::string random(where_array, where_array + sizeof(where_array)); std::string empty(""); std::string spaces(" "); try { @@ -38,8 +38,17 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { #endif for (const auto& where : wheres) { - boost::match_results what; - bool match = boost::regex_match(where, what, e, boost::match_default | boost::match_partial | boost::match_perl | boost::match_posix | boost::match_any); + try { + boost::match_results what; + bool match = boost::regex_match(where, what, e, boost::match_default | boost::match_partial | boost::match_posix | boost::match_any); + } catch(...) { + } + + try { + boost::match_results what; + bool match = boost::regex_match(where, what, e, boost::match_default | boost::match_partial | boost::match_perl | boost::match_any); + } catch(...) { + } } } catch(...) { } From 57fe4475324c5506adbfecdcdd2917f65c86ee9e Mon Sep 17 00:00:00 2001 From: DavidKorczynski Date: Sat, 14 Dec 2024 17:26:11 +0000 Subject: [PATCH 03/17] infra: chronos: extend local cache builder (#12851) Support validating if replay scripts work and whether ccache builds work Signed-off-by: David Korczynski --- .../experimental/chronos/build_cache_local.sh | 67 +++++++++++++++++-- 1 file changed, 62 insertions(+), 5 deletions(-) diff --git a/infra/experimental/chronos/build_cache_local.sh b/infra/experimental/chronos/build_cache_local.sh index 3a7e60a9930e..e116501d0086 100755 --- a/infra/experimental/chronos/build_cache_local.sh +++ b/infra/experimental/chronos/build_cache_local.sh @@ -24,6 +24,7 @@ cd projects/${_PROJECT} docker build -t gcr.io/oss-fuzz/${_PROJECT} . mkdir -p ccaches/${_PROJECT} +mkdir -p build/out/${_PROJECT} cd ${BASE} B_START=$SECONDS docker run \ @@ -31,12 +32,55 @@ docker run \ --env=SANITIZER=address \ --env=CCACHE_DIR=/workspace/ccache \ --env=FUZZING_LANGUAGE=${_FUZZING_LANGUAGE} \ + --env=CAPTURE_REPLAY_SCRIPT=1 \ --name=${_PROJECT}-origin-asan \ -v=$PWD/ccaches/${_PROJECT}/ccache:/workspace/ccache \ + -v=$PWD/build/out/${_PROJECT}/:/out/ \ gcr.io/oss-fuzz/${_PROJECT} \ -c \ "export PATH=/ccache/bin:\$PATH && compile" B_TIME=$(($SECONDS - $B_START)) +docker container commit ${_PROJECT}-origin-asan local/ossfuzz/${_PROJECT}-origin-asan +# Run the container with e.g. +# docker run --entrypoint /bin/bash -it local/ossfuzz/htslib-origin-asan +executables_vanilla="$(find ./build/out/${_PROJECT} -executable -type f | sort)" + +# Build with replay enabled, and validate the executables are the same +# in terms of naming. +R_START=$SECONDS +docker run \ + --entrypoint=/bin/bash \ + --env=SANITIZER=address \ + --env=REPLAY_ENABLED=1 \ + --env=FUZZING_LANGUAGE=${_FUZZING_LANGUAGE} \ + -v=$PWD/build/out/${_PROJECT}/:/out/ \ + --name=${_PROJECT}-origin-asan-replay-recached \ + local/ossfuzz/${_PROJECT}-origin-asan \ + -c \ + "export PATH=/ccache/bin:\$PATH && rm -rf /out/* && compile" +R_TIME=$(($SECONDS - $R_START)) + +executables_replay="$(find ./build/out/${_PROJECT}/ -executable -type f | sort)" + +echo "Executables vanilla: " +echo ${executables_vanilla} + +echo "------------------------------------------------------" +echo "Executables replay: " +echo ${executables_replay} + + +if [[ "$executables_replay" == "$executables_vanilla" ]] +then + echo "Replay worked" + echo "Vanilla compile time:" + echo ${B_TIME} + echo "Replay compile time:" + echo ${R_TIME} + exit 0 +else + echo "Replay did not work" +fi # Prepare Dockerfile for ccache cp -rf ccaches/${_PROJECT}/ccache ./projects/${_PROJECT}/ccache-cache @@ -53,13 +97,26 @@ docker run \ --env=SANITIZER=address \ --env=FUZZING_LANGUAGE=${_FUZZING_LANGUAGE} \ --name=${_PROJECT}-origin-asan-recached \ + -v=$PWD/build/out/${_PROJECT}/:/out/ \ us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-address \ -c \ - "export PATH=/ccache/bin:\$PATH && compile" + "export PATH=/ccache/bin:\$PATH && rm -rf /out/* && compile" A_TIME=$(($SECONDS - $A_START)) -echo "No cache: " -echo ${B_TIME} +executables_ccache="$(find ./build/out/${_PROJECT}/ -executable -type f | sort)" + +if [[ "$executables_ccache" == "$executables_vanilla" ]] +then + echo "Replaying failed, but ccache is working." + echo "No cache: " + echo ${B_TIME} + + echo "After cache: " + echo ${A_TIME} + + exit 0 +else + echo "Replay and ccaching did not work." +fi + -echo "After cache: " -echo ${A_TIME} From 9b8f61971d875b262d155ef3f93185fdeaea569a Mon Sep 17 00:00:00 2001 From: DavidKorczynski Date: Tue, 17 Dec 2024 10:47:48 +0000 Subject: [PATCH 04/17] cxxopts: fix build (#12856) Signed-off-by: David Korczynski --- projects/cxxopts/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/cxxopts/build.sh b/projects/cxxopts/build.sh index 577daee07d98..dd264f367e4c 100755 --- a/projects/cxxopts/build.sh +++ b/projects/cxxopts/build.sh @@ -15,4 +15,5 @@ # ################################################################################ +export USE_BAZEL_VERSION=7.4.0 bazel_build_fuzz_tests From ad5641addb85001e85340c67faf2a97c647c694b Mon Sep 17 00:00:00 2001 From: DavidKorczynski Date: Tue, 17 Dec 2024 10:48:02 +0000 Subject: [PATCH 05/17] faad2: fix build (#12857) Signed-off-by: David Korczynski --- projects/faad2/build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/faad2/build.sh b/projects/faad2/build.sh index 063ef6dc1376..dd264f367e4c 100644 --- a/projects/faad2/build.sh +++ b/projects/faad2/build.sh @@ -14,4 +14,6 @@ # limitations under the License. # ################################################################################ + +export USE_BAZEL_VERSION=7.4.0 bazel_build_fuzz_tests From 92c9fa207db307ac869d39d9584ec3510838da5a Mon Sep 17 00:00:00 2001 From: DavidKorczynski Date: Tue, 17 Dec 2024 10:48:17 +0000 Subject: [PATCH 06/17] abseil-cpp: fix build (#12858) Signed-off-by: David Korczynski --- projects/abseil-cpp/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/abseil-cpp/build.sh b/projects/abseil-cpp/build.sh index 3fd7d194d969..3896f0482787 100644 --- a/projects/abseil-cpp/build.sh +++ b/projects/abseil-cpp/build.sh @@ -14,6 +14,7 @@ # ################################################################################ +export USE_BAZEL_VERSION=7.4.0 # Disable `layering_check` feature. # As per https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=63223, it breaks # the build. Someone could figure out exactly why it breaks the build, but just From 3d393b174134650ab514bfd27ffd2fc0e7d7780d Mon Sep 17 00:00:00 2001 From: DavidKorczynski Date: Tue, 17 Dec 2024 11:09:55 +0000 Subject: [PATCH 07/17] chronos: extend timeout (#12855) --- infra/experimental/chronos/cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/experimental/chronos/cloudbuild.yaml b/infra/experimental/chronos/cloudbuild.yaml index f97c2567868b..2e4ae186b121 100644 --- a/infra/experimental/chronos/cloudbuild.yaml +++ b/infra/experimental/chronos/cloudbuild.yaml @@ -109,7 +109,7 @@ steps: images: - us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-address - us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-coverage -timeout: 1800s +timeout: 72000s # 20 hours, same as build_lib.py logsBucket: oss-fuzz-gcb-logs tags: - ${_PROJECT} From 02c5dc98843ada9a395d6c89cd00245fe76ea516 Mon Sep 17 00:00:00 2001 From: jonathanmetzman <31354670+jonathanmetzman@users.noreply.github.com> Date: Wed, 18 Dec 2024 01:50:53 +1100 Subject: [PATCH 08/17] Save docker images for C/C++ (#12831) So that bugs can reproduced. These should not be used if security is a concern! I maybe will put the word unsafe in the name --- infra/build/functions/build_lib.py | 36 ++++++++++++++----- infra/build/functions/build_project.py | 11 +++--- .../test_data/expected_build_steps.json | 10 ++++-- .../expected_centipede_build_steps.json | 7 ++-- .../expected_coverage_build_steps.json | 5 ++- .../test_data/expected_trial_build_steps.json | 5 ++- 6 files changed, 53 insertions(+), 21 deletions(-) diff --git a/infra/build/functions/build_lib.py b/infra/build/functions/build_lib.py index beb937c947bc..a975b2707ffc 100644 --- a/infra/build/functions/build_lib.py +++ b/infra/build/functions/build_lib.py @@ -415,6 +415,7 @@ def get_docker_build_step(image_names, 'name': DOCKER_TOOL_IMAGE, 'args': args, 'dir': directory, + 'id': f'build-{architecture}', } # Handle buildkit args # Note that we mutate "args" after making it a value in step. @@ -460,10 +461,11 @@ def get_project_image_steps( # pylint: disable=too-many-arguments steps.extend(get_pull_test_images_steps(config.test_image_suffix)) src_root = 'oss-fuzz' if not experiment else '.' - docker_build_step = get_docker_build_step([image], - os.path.join('projects', name), - src_root=src_root, - cache_image=cache_image) + docker_build_step = get_docker_build_step( + [image, _get_unsafe_name(name)], + os.path.join('projects', name), + src_root=src_root, + cache_image=cache_image) steps.append(docker_build_step) if srcmap: srcmap_step_id = get_srcmap_step_id() @@ -475,7 +477,7 @@ def get_project_image_steps( # pylint: disable=too-many-arguments ], 'env': [ 'OSSFUZZ_REVISION=$REVISION_ID', - 'FUZZING_LANGUAGE=%s' % language, + f'FUZZING_LANGUAGE={language}', ], 'id': srcmap_step_id }]) @@ -496,15 +498,31 @@ def get_project_image_steps( # pylint: disable=too-many-arguments 'args': ['buildx', 'use', builder_name] }, ]) - docker_build_arm_step = get_docker_build_step([image], - os.path.join( - 'projects', name), - architecture=_ARM64) + docker_build_arm_step = get_docker_build_step( + [image, _get_unsafe_name(name)], + os.path.join('projects', name), + architecture=_ARM64) steps.append(docker_build_arm_step) + if (not experiment and not config.testing and + config.build_type == 'fuzzing' and language in ('c', 'c++')): + # Push so that historical bugs are reproducible. + push_step = { + 'name': 'gcr.io/cloud-builders/docker', + 'args': ['push', _get_unsafe_name(name)], + 'id': 'push-image', + 'waitFor': [docker_build_step['id']], + 'allowFailure': True + } + steps.append(push_step) + return steps +def _get_unsafe_name(name): + return f'us-central1-docker.pkg.dev/oss-fuzz/unsafe/{name}' + + def get_logs_url(build_id): """Returns url that displays the build logs.""" return ( diff --git a/infra/build/functions/build_project.py b/infra/build/functions/build_project.py index b8eca8eb0190..2a6c4ad9b156 100755 --- a/infra/build/functions/build_project.py +++ b/infra/build/functions/build_project.py @@ -19,8 +19,6 @@ Usage: build_project.py """ -from __future__ import print_function - import argparse from dataclasses import dataclass import datetime @@ -77,6 +75,7 @@ class Config: experiment: bool = False # TODO(ochang): This should be different per engine+sanitizer combination. upload_build_logs: str = None + build_type: str = None WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)') @@ -476,7 +475,6 @@ def get_build_steps_for_project(project, upload_steps = get_upload_steps(project, build, timestamp, config.testing) build_steps.extend(upload_steps) - return build_steps @@ -629,7 +627,7 @@ def get_args(description): return parser.parse_args() -def create_config_from_commandline(args): +def create_config(args, build_type): """Create a Config object from parsed command line |args|.""" upload = not args.experiment return Config(testing=args.testing, @@ -637,7 +635,8 @@ def create_config_from_commandline(args): branch=args.branch, parallel=args.parallel, upload=upload, - experiment=args.experiment) + experiment=args.experiment, + build_type=build_type) def build_script_main(script_description, get_build_steps_func, build_type): @@ -650,7 +649,7 @@ def build_script_main(script_description, get_build_steps_func, build_type): credentials = oauth2client.client.GoogleCredentials.get_application_default() error = False - config = create_config_from_commandline(args) + config = create_config(args, build_type) for project_name in args.projects: logging.info('Getting steps for: "%s".', project_name) try: diff --git a/infra/build/functions/test_data/expected_build_steps.json b/infra/build/functions/test_data/expected_build_steps.json index 065c76c7bbbf..a35cb9a6ce80 100644 --- a/infra/build/functions/test_data/expected_build_steps.json +++ b/infra/build/functions/test_data/expected_build_steps.json @@ -14,9 +14,12 @@ "build", "--tag", "gcr.io/oss-fuzz/test-project", + "--tag", + "us-central1-docker.pkg.dev/oss-fuzz/unsafe/test-project", "." ], - "dir": "oss-fuzz/projects/test-project" + "dir": "oss-fuzz/projects/test-project", + "id": "build-x86_64" }, { "name": "gcr.io/oss-fuzz/test-project", @@ -68,9 +71,12 @@ "--load", "--tag", "gcr.io/oss-fuzz/test-project-aarch64", + "--tag", + "us-central1-docker.pkg.dev/oss-fuzz/unsafe/test-project-aarch64", "." ], - "dir": "oss-fuzz/projects/test-project" + "dir": "oss-fuzz/projects/test-project", + "id": "build-aarch64" }, { "name": "gcr.io/cloud-builders/docker", diff --git a/infra/build/functions/test_data/expected_centipede_build_steps.json b/infra/build/functions/test_data/expected_centipede_build_steps.json index db92d885ec2d..885c803d312a 100644 --- a/infra/build/functions/test_data/expected_centipede_build_steps.json +++ b/infra/build/functions/test_data/expected_centipede_build_steps.json @@ -13,10 +13,13 @@ "args": [ "build", "--tag", - "gcr.io/oss-fuzz/test-project", + "gcr.io/oss-fuzz/test-project", + "--tag", + "us-central1-docker.pkg.dev/oss-fuzz/unsafe/test-project", "." ], - "dir": "oss-fuzz/projects/test-project" + "dir": "oss-fuzz/projects/test-project", + "id": "build-x86_64" }, { "name": "gcr.io/oss-fuzz/test-project", diff --git a/infra/build/functions/test_data/expected_coverage_build_steps.json b/infra/build/functions/test_data/expected_coverage_build_steps.json index 9fce94c44bf2..493c6ce25d2a 100644 --- a/infra/build/functions/test_data/expected_coverage_build_steps.json +++ b/infra/build/functions/test_data/expected_coverage_build_steps.json @@ -14,9 +14,12 @@ "build", "--tag", "gcr.io/oss-fuzz/test-project", + "--tag", + "us-central1-docker.pkg.dev/oss-fuzz/unsafe/test-project", "." ], - "dir": "oss-fuzz/projects/test-project" + "dir": "oss-fuzz/projects/test-project", + "id": "build-x86_64" }, { "name": "gcr.io/oss-fuzz/test-project", diff --git a/infra/build/functions/test_data/expected_trial_build_steps.json b/infra/build/functions/test_data/expected_trial_build_steps.json index 1497f7e05a7a..a41da778516c 100644 --- a/infra/build/functions/test_data/expected_trial_build_steps.json +++ b/infra/build/functions/test_data/expected_trial_build_steps.json @@ -167,9 +167,12 @@ "build", "--tag", "gcr.io/oss-fuzz/skcms", + "--tag", + "us-central1-docker.pkg.dev/oss-fuzz/unsafe/skcms", "." ], - "dir": "oss-fuzz/projects/skcms" + "dir": "oss-fuzz/projects/skcms", + "id": "build-x86_64" }, { "name": "gcr.io/oss-fuzz/skcms", From 6ee533674ac01bfd5af9074cfa5f8702d8c500c5 Mon Sep 17 00:00:00 2001 From: DavidKorczynski Date: Tue, 17 Dec 2024 18:12:05 +0000 Subject: [PATCH 09/17] infra: enable introspector for Rust (#12854) Signed-off-by: David Korczynski --- infra/base-images/base-builder/compile | 59 +++++++++++++++++-- infra/base-images/base-clang/Dockerfile | 2 +- .../build/functions/build_and_run_coverage.py | 2 +- infra/build/functions/build_lib.py | 6 +- 4 files changed, 60 insertions(+), 9 deletions(-) diff --git a/infra/base-images/base-builder/compile b/infra/base-images/base-builder/compile index d0f45bb73fef..1c10d9e234e7 100755 --- a/infra/base-images/base-builder/compile +++ b/infra/base-images/base-builder/compile @@ -21,6 +21,9 @@ sysctl -w vm.mmap_rnd_bits=28 OSS_FUZZ_ON_DEMAND="${OSS_FUZZ_ON_DEMAND:-0}" +# Used for Rust introspector builds +RUST_SANITIZER=$SANITIZER + if [ "$FUZZING_LANGUAGE" = "jvm" ]; then if [ "$FUZZING_ENGINE" != "libfuzzer" ] && [ "$FUZZING_ENGINE" != "wycheproof" ]; then echo "ERROR: JVM projects can be fuzzed with libFuzzer or tested with wycheproof engines only." @@ -36,6 +39,15 @@ if [ "$FUZZING_LANGUAGE" = "jvm" ]; then fi fi +if [ "$FUZZING_LANGUAGE" = "rust" ]; then + if [ "$SANITIZER" = "introspector" ]; then + # introspector sanitizer flag will cause cargo build to fail. Rremove it + # temporarily, RUST_SANITIZER will hold the original sanitizer. + export SANITIZER=address + fi +fi + + if [ "$FUZZING_LANGUAGE" = "javascript" ]; then if [ "$FUZZING_ENGINE" != "libfuzzer" ]; then echo "ERROR: JavaScript projects can be fuzzed with libFuzzer engine only." @@ -111,7 +123,9 @@ fi # use RUSTFLAGS. # FIXME: Support code coverage once support is in. # See https://github.com/rust-lang/rust/issues/34701. -if [ "$SANITIZER" != "undefined" ] && [ "$SANITIZER" != "coverage" ] && [ "$SANITIZER" != "none" ] && [ "$ARCHITECTURE" != 'i386' ]; then +if [ "$RUST_SANITIZER" == "introspector" ]; then + export RUSTFLAGS="-Cdebuginfo=2 -Cforce-frame-pointers" +elif [ "$SANITIZER" != "undefined" ] && [ "$SANITIZER" != "coverage" ] && [ "$SANITIZER" != "none" ] && [ "$ARCHITECTURE" != 'i386' ]; then export RUSTFLAGS="--cfg fuzzing -Zsanitizer=${SANITIZER} -Cdebuginfo=1 -Cforce-frame-pointers" else export RUSTFLAGS="--cfg fuzzing -Cdebuginfo=1 -Cforce-frame-pointers" @@ -188,7 +202,7 @@ EOF export CXXFLAGS="$CXXFLAGS -fno-sanitize=leak" fi -if [ "$SANITIZER" = "introspector" ]; then +if [ "$SANITIZER" = "introspector" ] || [ "$RUST_SANITIZER" = "introspector" ]; then export AR=llvm-ar export NM=llvm-nm export RANLIB=llvm-ranlib @@ -210,10 +224,24 @@ if [ "$SANITIZER" = "introspector" ]; then apt-get install -y libjpeg-dev zlib1g-dev libyaml-dev python3 -m pip install --upgrade pip setuptools - python3 -m pip install cxxfilt pyyaml beautifulsoup4 lxml soupsieve + python3 -m pip install cxxfilt pyyaml beautifulsoup4 lxml soupsieve rust-demangler python3 -m pip install --prefer-binary matplotlib - python3 /fuzz-introspector/src/main.py light + # Install Fuzz-Introspector + pushd /fuzz-introspector/src + python3 -m pip install . + popd + + if [ "$FUZZING_LANGUAGE" = "python" ]; then + python3 /fuzz-introspector/src/main.py light --language=python + elif [ "$FUZZING_LANGUAGE" = "jvm" ]; then + python3 /fuzz-introspector/src/main.py light --language=jvm + elif [ "$FUZZING_LANGUAGE" = "rust" ]; then + python3 /fuzz-introspector/src/main.py light --language=rust + else + python3 /fuzz-introspector/src/main.py light + fi + rsync -avu --delete "$SRC/inspector/" "$OUT/inspector" fi @@ -280,7 +308,7 @@ else fi fi -if [ "$SANITIZER" = "introspector" ]; then +if [ "$SANITIZER" = "introspector" ] || [ "$RUST_SANITIZER" = "introspector" ]; then unset CXXFLAGS unset CFLAGS export G_ANALYTICS_TAG="G-8WTFM1Y62J" @@ -295,6 +323,21 @@ if [ "$SANITIZER" = "introspector" ]; then mkdir -p $SRC/my-fi-data find $OUT/ -name *.data -exec mv {} $SRC/my-fi-data/ \; find $OUT/ -name *.data.yaml -exec mv {} $SRC/my-fi-data/ \; + elif [ "$FUZZING_LANGUAGE" = "rust" ]; then + echo "GOING rust route" + + # Run the rust frontend + pushd /fuzz-introspector/frontends/rust/rust_function_analyser + cargo run -- $SRC + + # Move files temporarily to fix workflow of other languages. + mkdir -p $SRC/my-fi-data + find ./ -name "*.data" -exec mv {} $SRC/my-fi-data/ \; + find ./ -name "*.data.yaml" -exec mv {} $SRC/my-fi-data/ \; + popd + + # Restore the sanitizer flag for rust + export SANITIZER="introspector" fi mkdir -p $SRC/inspector @@ -335,6 +378,12 @@ if [ "$SANITIZER" = "introspector" ]; then REPORT_ARGS="$REPORT_ARGS --language=jvm" python3 /fuzz-introspector/src/main.py report $REPORT_ARGS rsync -avu --delete "$SRC/inspector/" "$OUT/inspector" + elif [ "$FUZZING_LANGUAGE" = "rust" ]; then + echo "GOING rust route" + REPORT_ARGS="$REPORT_ARGS --target_dir=$SRC/inspector" + REPORT_ARGS="$REPORT_ARGS --language=rust" + python3 /fuzz-introspector/src/main.py report $REPORT_ARGS + rsync -avu --delete "$SRC/inspector/" "$OUT/inspector" else # C/C++ diff --git a/infra/base-images/base-clang/Dockerfile b/infra/base-images/base-clang/Dockerfile index 8f6486896dcf..d403bb07d876 100644 --- a/infra/base-images/base-clang/Dockerfile +++ b/infra/base-images/base-clang/Dockerfile @@ -36,7 +36,7 @@ RUN apt-get update && apt-get install -y wget sudo && \ RUN apt-get update && apt-get install -y git && \ git clone https://github.com/ossf/fuzz-introspector.git fuzz-introspector && \ cd fuzz-introspector && \ - git checkout 5924aea8bcfe1fbdac9dc815adff91d3ee51f52b && \ + git checkout 74917384c5a4e368d900862b4bd3d16ce3fe5dd8 && \ git submodule init && \ git submodule update && \ apt-get autoremove --purge -y git && \ diff --git a/infra/build/functions/build_and_run_coverage.py b/infra/build/functions/build_and_run_coverage.py index 48ba786e8b1c..9ee40fd58a3e 100755 --- a/infra/build/functions/build_and_run_coverage.py +++ b/infra/build/functions/build_and_run_coverage.py @@ -46,7 +46,7 @@ 'c', 'c++', 'go', 'jvm', 'rust', 'swift', 'python' ] -LANGUAGES_WITH_INTROSPECTOR_SUPPORT = ['c', 'c++', 'python', 'jvm'] +LANGUAGES_WITH_INTROSPECTOR_SUPPORT = ['c', 'c++', 'python', 'jvm', 'rust'] class Bucket: # pylint: disable=too-few-public-methods diff --git a/infra/build/functions/build_lib.py b/infra/build/functions/build_lib.py index a975b2707ffc..f244a766e97e 100644 --- a/infra/build/functions/build_lib.py +++ b/infra/build/functions/build_lib.py @@ -275,11 +275,13 @@ def download_coverage_data_steps(project_name, latest, bucket_name, out_dir): bucket_url = f'gs://{bucket_name}/{project_name}/textcov_reports/{latest}/*' steps.append({ 'name': 'gcr.io/cloud-builders/gsutil', - 'args': ['-m', 'cp', '-r', bucket_url, coverage_data_path] + 'args': ['-m', 'cp', '-r', bucket_url, coverage_data_path], + 'allowFailure': True }) steps.append({ 'name': 'gcr.io/oss-fuzz-base/base-runner', - 'args': ['bash', '-c', f'ls -lrt {out_dir}/textcov_reports'] + 'args': ['bash', '-c', f'ls -lrt {out_dir}/textcov_reports'], + 'allowFailure': True }) return steps From 548d17a218ef58dfc7b745462508b562f374d232 Mon Sep 17 00:00:00 2001 From: AdamKorcz <44787359+AdamKorcz@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:24:18 +0000 Subject: [PATCH 10/17] distribution: fix broken build (#12860) Signed-off-by: Adam Korczynski --- projects/distribution/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/distribution/Dockerfile b/projects/distribution/Dockerfile index 1c9170301e43..f4317ceb6e00 100644 --- a/projects/distribution/Dockerfile +++ b/projects/distribution/Dockerfile @@ -17,5 +17,6 @@ FROM gcr.io/oss-fuzz-base/base-builder-go RUN git clone --depth 1 https://github.com/distribution/distribution RUN git clone --depth 1 https://github.com/cncf/cncf-fuzzing +RUN git clone --depth 1 https://github.com/AdamKorcz/go-118-fuzz-build --branch=november-backup COPY build.sh $SRC/ WORKDIR $SRC/distribution From 4161b2267f39e32f32e122501b7b82f9bc28caea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Novomesk=C3=BD?= <52529860+novomesk@users.noreply.github.com> Date: Wed, 18 Dec 2024 00:41:29 +0100 Subject: [PATCH 11/17] kimageformats: upgrade libaom, openjpeg, libjxl (#12861) --- projects/kimageformats/Dockerfile | 6 +++--- projects/kimageformats/build.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/kimageformats/Dockerfile b/projects/kimageformats/Dockerfile index 1e486ca00245..72aa8d01d114 100644 --- a/projects/kimageformats/Dockerfile +++ b/projects/kimageformats/Dockerfile @@ -26,12 +26,12 @@ RUN git clone --depth 1 -b master https://invent.kde.org/frameworks/extra-cmake- RUN git clone --depth 1 --branch=dev git://code.qt.io/qt/qtbase.git RUN git clone --depth 1 -b master https://invent.kde.org/frameworks/karchive.git RUN git clone --depth 1 -b master https://invent.kde.org/frameworks/kimageformats.git -RUN git clone --depth 1 -b v3.9.1 https://aomedia.googlesource.com/aom +RUN git clone --depth 1 -b v3.11.0 https://aomedia.googlesource.com/aom RUN git clone --depth 1 -b v1.1.1 https://github.com/AOMediaCodec/libavif.git RUN git clone --depth 1 https://github.com/strukturag/libde265.git -RUN git clone --depth 1 -b v2.5.2 https://github.com/uclouvain/openjpeg.git +RUN git clone --depth 1 -b v2.5.3 https://github.com/uclouvain/openjpeg.git RUN git clone --depth 1 https://github.com/strukturag/libheif.git -RUN git clone --depth=1 --branch v0.10.x --recursive --shallow-submodules https://github.com/libjxl/libjxl.git +RUN git clone --depth=1 --branch v0.11.x --recursive --shallow-submodules https://github.com/libjxl/libjxl.git RUN git clone --depth 1 https://github.com/LibRaw/LibRaw RUN git clone --depth 1 https://github.com/mircomir/jxrlib.git COPY build.sh $SRC diff --git a/projects/kimageformats/build.sh b/projects/kimageformats/build.sh index dfbf942ae6c6..f9bec5ae4fe4 100644 --- a/projects/kimageformats/build.sh +++ b/projects/kimageformats/build.sh @@ -152,7 +152,7 @@ sed -i "s/static const int MAX_IMAGE_WIDTH = 32768;/static const int MAX_IMAGE_W sed -i "s/static const int MAX_IMAGE_HEIGHT = 32768;/static const int MAX_IMAGE_HEIGHT = 8192;/g" libheif/security_limits.h mkdir build cd build -cmake -DBUILD_SHARED_LIBS=OFF -DENABLE_PLUGIN_LOADING=OFF -DWITH_DAV1D=OFF -DWITH_EXAMPLES=OFF -DWITH_LIBDE265=ON -DWITH_RAV1E=OFF -DWITH_RAV1E_PLUGIN=OFF -DWITH_SvtEnc=OFF -DWITH_SvtEnc_PLUGIN=OFF -DWITH_X265=OFF -DWITH_OpenJPEG_DECODER=ON .. +cmake -DBUILD_SHARED_LIBS=OFF -DENABLE_PLUGIN_LOADING=OFF -DWITH_DAV1D=OFF -DWITH_EXAMPLES=OFF -DWITH_LIBDE265=ON -DWITH_RAV1E=OFF -DWITH_RAV1E_PLUGIN=OFF -DWITH_SvtEnc=OFF -DWITH_SvtEnc_PLUGIN=OFF -DWITH_X265=OFF -DWITH_OpenJPEG_DECODER=ON -DWITH_OpenH264_DECODER=OFF .. make -j$(nproc) make install -j$(nproc) From ebfb0984af802624db758c437f67a0dc3e34b228 Mon Sep 17 00:00:00 2001 From: DavidKorczynski Date: Wed, 18 Dec 2024 10:15:04 +0000 Subject: [PATCH 12/17] infra: chronos: improve local cache building and explicitly outline steps (#12862) Signed-off-by: David Korczynski --- infra/experimental/chronos/README.md | 37 ++++++------- .../experimental/chronos/build_cache_local.sh | 54 +++++++++++++++---- 2 files changed, 60 insertions(+), 31 deletions(-) diff --git a/infra/experimental/chronos/README.md b/infra/experimental/chronos/README.md index 9f6a0e1846d4..4dfe1baeed8a 100644 --- a/infra/experimental/chronos/README.md +++ b/infra/experimental/chronos/README.md @@ -1,23 +1,20 @@ -# Usage -Under `OSS-Fuzz` root directory: -```bash -export PROJECT=libiec61850 -export FUZZ_TARGET=fuzz_mms_decode.c -export FUZZING_LANGUAGE=c +# Chronos: rebuilding OSS-Fuzz harnesses using cached builds -infra/experimental/chronos/prepare-recompile "$PROJECT" "$FUZZ_TARGET" "$FUZZING_LANGUAGE" -python infra/helper.py build_image "$PROJECT" -# AddressSanitizer. -docker run -ti --entrypoint="/bin/sh" --env SANITIZER="address" --name "${PROJECT}-origin-asan" "gcr.io/oss-fuzz/${PROJECT}" -c "compile && rm -rf /out/*" -docker commit --change 'CMD ["compile"] --change 'ENTRYPOINT /bin/sh' "${PROJECT}-origin-asan" "gcr.io/oss-fuzz/${PROJECT}-ofg-cached-asan" -docker run -ti --entrypoint="recompile" "gcr.io/oss-fuzz/${PROJECT}-ofg-cached-asan" +## Usage locally -# Coverage measurement. -docker run -ti --entrypoint="/bin/sh" --env SANITIZER="coverage" --name "${PROJECT}-origin-cov" "gcr.io/oss-fuzz/${PROJECT}" -c "compile && rm -rf /out/*" -docker commit --change 'CMD ["compile"] --change 'ENTRYPOINT /bin/sh' "${PROJECT}-origin-cov" "gcr.io/oss-fuzz/${PROJECT}-ofg-cached-cov" -docker run -ti --entrypoint="recompile" "gcr.io/oss-fuzz/${PROJECT}-ofg-cached-cov" -``` +**Example 1: htslib** + +From the OSS-Fuzz root -# Assumptions -1. Fuzzer: Chronos assumes `libFuzzer`. Other fuzzers are not well-supported, but may work by setting ENV `FUZZING_ENGINE` in project's `Dockerfile`. -2. Sanitizer: Chronos assumes `AddressSanitizer`. Other sanitizers may work by adding setting ENV `SANITIZER` in project's `Dockerfile`. +```sh +$ RUN_ALL=1 ./infra/experimental/chronos/build_cache_local.sh htslib c +... +... +Vanilla compile time: +17 +Replay worked +Replay compile time: +2 +Ccache compile time: +9 +``` diff --git a/infra/experimental/chronos/build_cache_local.sh b/infra/experimental/chronos/build_cache_local.sh index e116501d0086..df2edba312cc 100755 --- a/infra/experimental/chronos/build_cache_local.sh +++ b/infra/experimental/chronos/build_cache_local.sh @@ -20,12 +20,16 @@ _FUZZING_LANGUAGE=$2 BASE=$PWD +# Step 1: build the base image cd projects/${_PROJECT} docker build -t gcr.io/oss-fuzz/${_PROJECT} . + +# Step 2: create a container where `compile` has run which enables ccaching +# and also generates a replay build script. +cd ${BASE} mkdir -p ccaches/${_PROJECT} mkdir -p build/out/${_PROJECT} -cd ${BASE} B_START=$SECONDS docker run \ --entrypoint=/bin/bash \ @@ -40,13 +44,22 @@ docker run \ -c \ "export PATH=/ccache/bin:\$PATH && compile" B_TIME=$(($SECONDS - $B_START)) + +# Step 3: save (commit, locally) the cached container as an image docker container commit ${_PROJECT}-origin-asan local/ossfuzz/${_PROJECT}-origin-asan -# Run the container with e.g. -# docker run --entrypoint /bin/bash -it local/ossfuzz/htslib-origin-asan + + +# Step 4: save the list of executables created from a vanilla build. This is +# needed for validating if replay and ccaching works. +# notes: run a shell the container with e.g. +# `docker run --entrypoint /bin/bash -it local/ossfuzz/htslib-origin-asan` executables_vanilla="$(find ./build/out/${_PROJECT} -executable -type f | sort)" -# Build with replay enabled, and validate the executables are the same + +# Step 5: Build with replay enabled, and validate the executables are the same # in terms of naming. +# Note that an important step is removing everything in $OUT/ which is done +# in the docker command. R_START=$SECONDS docker run \ --entrypoint=/bin/bash \ @@ -60,6 +73,7 @@ docker run \ "export PATH=/ccache/bin:\$PATH && rm -rf /out/* && compile" R_TIME=$(($SECONDS - $R_START)) +# Step 6: Extract the newly build executables executables_replay="$(find ./build/out/${_PROJECT}/ -executable -type f | sort)" echo "Executables vanilla: " @@ -69,7 +83,8 @@ echo "------------------------------------------------------" echo "Executables replay: " echo ${executables_replay} - +# Step 7: match executables from vanilla builds and replay builds. +# If this step is successful, then the process can exit as it's ready. if [[ "$executables_replay" == "$executables_vanilla" ]] then echo "Replay worked" @@ -77,19 +92,28 @@ then echo ${B_TIME} echo "Replay compile time:" echo ${R_TIME} - exit 0 + + if [ -n "${RUN_ALL+1}" ]; then + exit 0 + fi else echo "Replay did not work" fi -# Prepare Dockerfile for ccache +# Step 8: prepare Dockerfile for ccache cp -rf ccaches/${_PROJECT}/ccache ./projects/${_PROJECT}/ccache-cache infra/experimental/chronos/prepare-ccache ${_PROJECT} cd projects/${_PROJECT} + +# Step 9: Build an image with CCache's new items (modifications are done on the +# dockerfile) docker build -t us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/${_PROJECT}-ofg-cached-address . +cd ${BASE} + +# Step 10: Run a `compile` with ccache's image. # Run the ccache build A_START=$SECONDS docker run \ @@ -103,15 +127,23 @@ docker run \ "export PATH=/ccache/bin:\$PATH && rm -rf /out/* && compile" A_TIME=$(($SECONDS - $A_START)) +# Step 11: extract the executables from the ccache build executables_ccache="$(find ./build/out/${_PROJECT}/ -executable -type f | sort)" + +# Step 12: validate the ccache builds are successful if [[ "$executables_ccache" == "$executables_vanilla" ]] then - echo "Replaying failed, but ccache is working." - echo "No cache: " + echo "Vanilla compile time:" echo ${B_TIME} - - echo "After cache: " + if [[ "$executables_replay" == "$executables_vanilla" ]] + then + echo "Replay worked" + echo "Replay compile time:" + echo ${R_TIME} + fi + + echo "Ccache compile time: " echo ${A_TIME} exit 0 From 0c42f314831acbd47357df89b777696711f52771 Mon Sep 17 00:00:00 2001 From: AdamKorcz <44787359+AdamKorcz@users.noreply.github.com> Date: Wed, 18 Dec 2024 10:25:08 +0000 Subject: [PATCH 13/17] cri-o: fix broken build (#12865) Signed-off-by: Adam Korczynski --- projects/cri-o/Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/projects/cri-o/Dockerfile b/projects/cri-o/Dockerfile index 4f062be25fef..67f2d0f302c5 100644 --- a/projects/cri-o/Dockerfile +++ b/projects/cri-o/Dockerfile @@ -14,11 +14,17 @@ # ################################################################################ +FROM gcr.io/oss-fuzz-base/base-builder FROM gcr.io/oss-fuzz-base/base-builder-go RUN apt-get update && apt-get install -y libaio-dev autoconf gettext texinfo \ libbtrfs-dev git libassuan-dev libdevmapper-dev libglib2.0-dev libc6-dev \ libgpgme-dev libgpg-error-dev libseccomp-dev libsystemd-dev libselinux1-dev \ pkg-config go-md2man libudev-dev software-properties-common systemd +RUN wget https://go.dev/dl/go1.23.4.linux-amd64.tar.gz \ + && mkdir temp-go \ + && rm -rf /root/.go/* \ + && tar -C temp-go/ -xzf go1.23.4.linux-amd64.tar.gz \ + && mv temp-go/go/* /root/.go/ RUN git clone --depth 1 https://github.com/cri-o/cri-o RUN git clone --depth 1 https://github.com/cncf/cncf-fuzzing COPY build.sh $SRC/ From f5ece1dfc6c5627e0dd19e43ddd7be1bc2e2319d Mon Sep 17 00:00:00 2001 From: AdamKorcz <44787359+AdamKorcz@users.noreply.github.com> Date: Wed, 18 Dec 2024 14:21:47 +0000 Subject: [PATCH 14/17] etcd: fix broken build (#12866) Signed-off-by: Adam Korczynski --- projects/etcd/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/projects/etcd/Dockerfile b/projects/etcd/Dockerfile index 88093fcfc3e5..c69c26832bc3 100644 --- a/projects/etcd/Dockerfile +++ b/projects/etcd/Dockerfile @@ -18,5 +18,10 @@ FROM gcr.io/oss-fuzz-base/base-builder-go RUN git clone --depth 1 https://github.com/etcd-io/etcd RUN git clone --depth 1 https://github.com/etcd-io/raft RUN git clone --depth 1 https://github.com/cncf/cncf-fuzzing +RUN wget https://go.dev/dl/go1.23.4.linux-amd64.tar.gz \ + && mkdir temp-go \ + && rm -rf /root/.go/* \ + && tar -C temp-go/ -xzf go1.23.4.linux-amd64.tar.gz \ + && mv temp-go/go/* /root/.go/ COPY build.sh $SRC/ WORKDIR $SRC/etcd From 411b8a86d5d506dac5bd8ef24994f67235f13d3b Mon Sep 17 00:00:00 2001 From: AdamKorcz <44787359+AdamKorcz@users.noreply.github.com> Date: Wed, 18 Dec 2024 15:13:20 +0000 Subject: [PATCH 15/17] dgraph: fix broken build (#12867) Signed-off-by: Adam Korczynski --- projects/dgraph/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/dgraph/build.sh b/projects/dgraph/build.sh index 184a2217617c..6bdec5f719d5 100644 --- a/projects/dgraph/build.sh +++ b/projects/dgraph/build.sh @@ -18,4 +18,4 @@ mv $SRC/fuzz_parser_test.go $SRC/dgraph/dql/ printf "package dql\nimport _ \"github.com/AdamKorcz/go-118-fuzz-build/testing\"\n" > dql/register.go go mod tidy -compile_native_go_fuzzer github.com/dgraph-io/dgraph/dql FuzzParserTest parser_fuzzer +compile_native_go_fuzzer github.com/dgraph-io/dgraph/v24/dql FuzzParserTest parser_fuzzer From f7a14bc67e4306dc277f79a1fb93798a67058ab9 Mon Sep 17 00:00:00 2001 From: Ryan Fox-Tyler <60440289+ryanfoxtyler@users.noreply.github.com> Date: Wed, 18 Dec 2024 10:35:18 -0500 Subject: [PATCH 16/17] update Dgraph project.yaml (#12775) update email domains from dgraph.io -> hypermode.com (Hypermode acquired Dgraph last year) and add additional engineer --- projects/dgraph/project.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/projects/dgraph/project.yaml b/projects/dgraph/project.yaml index 72199ee5745e..04796f4a2783 100644 --- a/projects/dgraph/project.yaml +++ b/projects/dgraph/project.yaml @@ -1,9 +1,10 @@ homepage: "https://dgraph.io" main_repo: "https://github.com/dgraph-io/dgraph" -primary_contact: "security@dgraph.io" +primary_contact: "security@hypermode.com" auto_ccs : - - "harshil@dgraph.io" - - "ryan@dgraph.io" + - "harshil@hypermode.com" + - "ryan@hypermode.com" + - "aman@hypermode.com" language: go fuzzing_engines: - libfuzzer From 35292ac78bd2521052f89dd10be5bc67232cbb71 Mon Sep 17 00:00:00 2001 From: AdamKorcz <44787359+AdamKorcz@users.noreply.github.com> Date: Wed, 18 Dec 2024 16:03:10 +0000 Subject: [PATCH 17/17] istio: fix broken build (#12868) Signed-off-by: Adam Korczynski --- projects/istio/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/projects/istio/Dockerfile b/projects/istio/Dockerfile index e9ddcdd95946..73137fc7222a 100644 --- a/projects/istio/Dockerfile +++ b/projects/istio/Dockerfile @@ -16,6 +16,11 @@ # Setup the builder for Istio. The standard Go builder is sufficient. FROM gcr.io/oss-fuzz-base/base-builder-go +RUN wget https://go.dev/dl/go1.23.4.linux-amd64.tar.gz \ + && mkdir temp-go \ + && rm -rf /root/.go/* \ + && tar -C temp-go/ -xzf go1.23.4.linux-amd64.tar.gz \ + && mv temp-go/go/* /root/.go/ RUN git clone --depth 1 https://github.com/istio/istio COPY build.sh $SRC/ WORKDIR $SRC/istio