Merge branch 'master' into go-ngolo-1229

infra/base-images/base-builder/compile

@@ -21,6 +21,9 @@ sysctl -w vm.mmap_rnd_bits=28
 
 OSS_FUZZ_ON_DEMAND="${OSS_FUZZ_ON_DEMAND:-0}"
 
+# Used for Rust introspector builds
+RUST_SANITIZER=$SANITIZER
+
 if [ "$FUZZING_LANGUAGE" = "jvm" ]; then
   if [ "$FUZZING_ENGINE" != "libfuzzer" ] && [ "$FUZZING_ENGINE" != "wycheproof" ]; then
     echo "ERROR: JVM projects can be fuzzed with libFuzzer or tested with wycheproof engines only."
@@ -36,6 +39,15 @@ if [ "$FUZZING_LANGUAGE" = "jvm" ]; then
   fi
 fi
 
+if [ "$FUZZING_LANGUAGE" = "rust" ]; then
+  if [ "$SANITIZER" = "introspector" ]; then
+    # introspector sanitizer flag will cause cargo build to fail. Rremove it
+    # temporarily, RUST_SANITIZER will hold the original sanitizer.
+    export SANITIZER=address
+  fi
+fi
+
+
 if [ "$FUZZING_LANGUAGE" = "javascript" ]; then
   if [ "$FUZZING_ENGINE" != "libfuzzer" ]; then
     echo "ERROR: JavaScript projects can be fuzzed with libFuzzer engine only."
@@ -111,7 +123,9 @@ fi
 # use RUSTFLAGS.
 # FIXME: Support code coverage once support is in.
 # See https://github.com/rust-lang/rust/issues/34701.
-if [ "$SANITIZER" != "undefined" ] && [ "$SANITIZER" != "coverage" ] && [ "$SANITIZER" != "none" ] && [ "$ARCHITECTURE" != 'i386' ]; then
+if [ "$RUST_SANITIZER" == "introspector" ]; then
+  export RUSTFLAGS="-Cdebuginfo=2 -Cforce-frame-pointers"
+elif [ "$SANITIZER" != "undefined" ] && [ "$SANITIZER" != "coverage" ] && [ "$SANITIZER" != "none" ] && [ "$ARCHITECTURE" != 'i386' ]; then
   export RUSTFLAGS="--cfg fuzzing -Zsanitizer=${SANITIZER} -Cdebuginfo=1 -Cforce-frame-pointers"
 else
   export RUSTFLAGS="--cfg fuzzing -Cdebuginfo=1 -Cforce-frame-pointers"
@@ -188,7 +202,7 @@ EOF
   export CXXFLAGS="$CXXFLAGS -fno-sanitize=leak"
 fi
 
-if [ "$SANITIZER" = "introspector" ]; then
+if [ "$SANITIZER" = "introspector" ] || [ "$RUST_SANITIZER" = "introspector" ]; then
   export AR=llvm-ar
   export NM=llvm-nm
   export RANLIB=llvm-ranlib
@@ -210,10 +224,24 @@ if [ "$SANITIZER" = "introspector" ]; then
 
   apt-get install -y libjpeg-dev zlib1g-dev libyaml-dev
   python3 -m pip install --upgrade pip setuptools
-  python3 -m pip install cxxfilt pyyaml beautifulsoup4 lxml soupsieve
+  python3 -m pip install cxxfilt pyyaml beautifulsoup4 lxml soupsieve rust-demangler
   python3 -m pip install --prefer-binary matplotlib
 
-  python3 /fuzz-introspector/src/main.py light
+  # Install Fuzz-Introspector
+  pushd /fuzz-introspector/src
+  python3 -m pip install .
+  popd
+
+  if [ "$FUZZING_LANGUAGE" = "python" ]; then
+    python3 /fuzz-introspector/src/main.py light --language=python
+  elif [ "$FUZZING_LANGUAGE" = "jvm" ]; then
+    python3 /fuzz-introspector/src/main.py light --language=jvm
+  elif [ "$FUZZING_LANGUAGE" = "rust" ]; then
+    python3 /fuzz-introspector/src/main.py light --language=rust
+  else
+    python3 /fuzz-introspector/src/main.py light
+  fi
+
   rsync -avu --delete "$SRC/inspector/" "$OUT/inspector"
 fi
 
@@ -280,7 +308,7 @@ else
   fi
 fi
 
-if [ "$SANITIZER" = "introspector" ]; then
+if [ "$SANITIZER" = "introspector" ] || [ "$RUST_SANITIZER" = "introspector" ]; then
   unset CXXFLAGS
   unset CFLAGS
   export G_ANALYTICS_TAG="G-8WTFM1Y62J"
@@ -295,6 +323,21 @@ if [ "$SANITIZER" = "introspector" ]; then
     mkdir -p $SRC/my-fi-data
     find $OUT/ -name *.data -exec mv {} $SRC/my-fi-data/ \;
     find $OUT/ -name *.data.yaml -exec mv {} $SRC/my-fi-data/ \;
+  elif [ "$FUZZING_LANGUAGE" = "rust" ]; then
+    echo "GOING rust route"
+
+    # Run the rust frontend
+    pushd /fuzz-introspector/frontends/rust/rust_function_analyser
+    cargo run -- $SRC
+
+    # Move files temporarily to fix workflow of other languages.
+    mkdir -p $SRC/my-fi-data
+    find ./ -name "*.data" -exec mv {} $SRC/my-fi-data/ \;
+    find ./ -name "*.data.yaml" -exec mv {} $SRC/my-fi-data/ \;
+    popd
+
+    # Restore the sanitizer flag for rust
+    export SANITIZER="introspector"
   fi
 
   mkdir -p $SRC/inspector
@@ -335,6 +378,12 @@ if [ "$SANITIZER" = "introspector" ]; then
     REPORT_ARGS="$REPORT_ARGS --language=jvm"
     python3 /fuzz-introspector/src/main.py report $REPORT_ARGS
     rsync -avu --delete "$SRC/inspector/" "$OUT/inspector"
+  elif [ "$FUZZING_LANGUAGE" = "rust" ]; then
+    echo "GOING rust route"
+    REPORT_ARGS="$REPORT_ARGS --target_dir=$SRC/inspector"
+    REPORT_ARGS="$REPORT_ARGS --language=rust"
+    python3 /fuzz-introspector/src/main.py report $REPORT_ARGS
+    rsync -avu --delete "$SRC/inspector/" "$OUT/inspector"
   else
     # C/C++
 

infra/base-images/base-clang/Dockerfile

@@ -36,7 +36,7 @@ RUN apt-get update && apt-get install -y wget sudo && \
 RUN apt-get update && apt-get install -y git && \
     git clone https://github.com/ossf/fuzz-introspector.git fuzz-introspector && \
     cd fuzz-introspector && \
-    git checkout 5924aea8bcfe1fbdac9dc815adff91d3ee51f52b && \
+    git checkout 74917384c5a4e368d900862b4bd3d16ce3fe5dd8 && \
     git submodule init && \
     git submodule update && \
     apt-get autoremove --purge -y git && \

infra/build/functions/build_and_run_coverage.py

@@ -46,7 +46,7 @@
     'c', 'c++', 'go', 'jvm', 'rust', 'swift', 'python'
 ]
 
-LANGUAGES_WITH_INTROSPECTOR_SUPPORT = ['c', 'c++', 'python', 'jvm']
+LANGUAGES_WITH_INTROSPECTOR_SUPPORT = ['c', 'c++', 'python', 'jvm', 'rust']
 
 
 class Bucket:  # pylint: disable=too-few-public-methods

infra/build/functions/build_lib.py

@@ -275,11 +275,13 @@ def download_coverage_data_steps(project_name, latest, bucket_name, out_dir):
   bucket_url = f'gs://{bucket_name}/{project_name}/textcov_reports/{latest}/*'
   steps.append({
       'name': 'gcr.io/cloud-builders/gsutil',
-      'args': ['-m', 'cp', '-r', bucket_url, coverage_data_path]
+      'args': ['-m', 'cp', '-r', bucket_url, coverage_data_path],
+      'allowFailure': True
   })
   steps.append({
       'name': 'gcr.io/oss-fuzz-base/base-runner',
-      'args': ['bash', '-c', f'ls -lrt {out_dir}/textcov_reports']
+      'args': ['bash', '-c', f'ls -lrt {out_dir}/textcov_reports'],
+      'allowFailure': True
   })
 
   return steps
@@ -415,6 +417,7 @@ def get_docker_build_step(image_names,
       'name': DOCKER_TOOL_IMAGE,
       'args': args,
       'dir': directory,
+      'id': f'build-{architecture}',
   }
   # Handle buildkit args
   # Note that we mutate "args" after making it a value in step.
@@ -460,10 +463,11 @@ def get_project_image_steps(  # pylint: disable=too-many-arguments
     steps.extend(get_pull_test_images_steps(config.test_image_suffix))
   src_root = 'oss-fuzz' if not experiment else '.'
 
-  docker_build_step = get_docker_build_step([image],
-                                            os.path.join('projects', name),
-                                            src_root=src_root,
-                                            cache_image=cache_image)
+  docker_build_step = get_docker_build_step(
+      [image, _get_unsafe_name(name)],
+      os.path.join('projects', name),
+      src_root=src_root,
+      cache_image=cache_image)
   steps.append(docker_build_step)
   if srcmap:
     srcmap_step_id = get_srcmap_step_id()
@@ -475,7 +479,7 @@ def get_project_image_steps(  # pylint: disable=too-many-arguments
         ],
         'env': [
             'OSSFUZZ_REVISION=$REVISION_ID',
-            'FUZZING_LANGUAGE=%s' % language,
+            f'FUZZING_LANGUAGE={language}',
         ],
         'id': srcmap_step_id
     }])
@@ -496,15 +500,31 @@ def get_project_image_steps(  # pylint: disable=too-many-arguments
             'args': ['buildx', 'use', builder_name]
         },
     ])
-    docker_build_arm_step = get_docker_build_step([image],
-                                                  os.path.join(
-                                                      'projects', name),
-                                                  architecture=_ARM64)
+    docker_build_arm_step = get_docker_build_step(
+        [image, _get_unsafe_name(name)],
+        os.path.join('projects', name),
+        architecture=_ARM64)
     steps.append(docker_build_arm_step)
 
+  if (not experiment and not config.testing and
+      config.build_type == 'fuzzing' and language in ('c', 'c++')):
+    # Push so that historical bugs are reproducible.
+    push_step = {
+        'name': 'gcr.io/cloud-builders/docker',
+        'args': ['push', _get_unsafe_name(name)],
+        'id': 'push-image',
+        'waitFor': [docker_build_step['id']],
+        'allowFailure': True
+    }
+    steps.append(push_step)
+
   return steps
 
 
+def _get_unsafe_name(name):
+  return f'us-central1-docker.pkg.dev/oss-fuzz/unsafe/{name}'
+
+
 def get_logs_url(build_id):
   """Returns url that displays the build logs."""
   return (

infra/build/functions/build_project.py

@@ -19,8 +19,6 @@
 Usage: build_project.py <project_dir>
 """
 
-from __future__ import print_function
-
 import argparse
 from dataclasses import dataclass
 import datetime
@@ -77,6 +75,7 @@ class Config:
   experiment: bool = False
   # TODO(ochang): This should be different per engine+sanitizer combination.
   upload_build_logs: str = None
+  build_type: str = None
 
 
 WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)')
@@ -476,7 +475,6 @@ def get_build_steps_for_project(project,
           upload_steps = get_upload_steps(project, build, timestamp,
                                           config.testing)
           build_steps.extend(upload_steps)
-
   return build_steps
 
 
@@ -629,15 +627,16 @@ def get_args(description):
   return parser.parse_args()
 
 
-def create_config_from_commandline(args):
+def create_config(args, build_type):
   """Create a Config object from parsed command line |args|."""
   upload = not args.experiment
   return Config(testing=args.testing,
                 test_image_suffix=args.test_image_suffix,
                 branch=args.branch,
                 parallel=args.parallel,
                 upload=upload,
-                experiment=args.experiment)
+                experiment=args.experiment,
+                build_type=build_type)
 
 
 def build_script_main(script_description, get_build_steps_func, build_type):
@@ -650,7 +649,7 @@ def build_script_main(script_description, get_build_steps_func, build_type):
 
   credentials = oauth2client.client.GoogleCredentials.get_application_default()
   error = False
-  config = create_config_from_commandline(args)
+  config = create_config(args, build_type)
   for project_name in args.projects:
     logging.info('Getting steps for: "%s".', project_name)
     try:

infra/build/functions/test_data/expected_build_steps.json

@@ -14,9 +14,12 @@
       "build",
       "--tag",
       "gcr.io/oss-fuzz/test-project",
+      "--tag",
+      "us-central1-docker.pkg.dev/oss-fuzz/unsafe/test-project",
       "."
     ],
-    "dir": "oss-fuzz/projects/test-project"
+    "dir": "oss-fuzz/projects/test-project",
+    "id": "build-x86_64"
   },
   {
     "name": "gcr.io/oss-fuzz/test-project",
@@ -68,9 +71,12 @@
       "--load",
       "--tag",
       "gcr.io/oss-fuzz/test-project-aarch64",
+      "--tag",
+      "us-central1-docker.pkg.dev/oss-fuzz/unsafe/test-project-aarch64",
       "."
     ],
-    "dir": "oss-fuzz/projects/test-project"
+    "dir": "oss-fuzz/projects/test-project",
+    "id": "build-aarch64"
   },
   {
     "name": "gcr.io/cloud-builders/docker",

infra/build/functions/test_data/expected_centipede_build_steps.json

@@ -13,10 +13,13 @@
     "args": [
       "build",
       "--tag",
-      "gcr.io/oss-fuzz/test-project",
+	"gcr.io/oss-fuzz/test-project",
+      "--tag",
+	"us-central1-docker.pkg.dev/oss-fuzz/unsafe/test-project",
       "."
     ],
-    "dir": "oss-fuzz/projects/test-project"
+    "dir": "oss-fuzz/projects/test-project",
+    "id": "build-x86_64"
   },
   {
     "name": "gcr.io/oss-fuzz/test-project",

infra/build/functions/test_data/expected_coverage_build_steps.json

@@ -14,9 +14,12 @@
       "build",
       "--tag",
       "gcr.io/oss-fuzz/test-project",
+      "--tag",
+	"us-central1-docker.pkg.dev/oss-fuzz/unsafe/test-project",
       "."
     ],
-    "dir": "oss-fuzz/projects/test-project"
+    "dir": "oss-fuzz/projects/test-project",
+    "id": "build-x86_64"
   },
   {
     "name": "gcr.io/oss-fuzz/test-project",

infra/build/functions/test_data/expected_trial_build_steps.json

@@ -167,9 +167,12 @@
       "build",
       "--tag",
       "gcr.io/oss-fuzz/skcms",
+      "--tag",
+      "us-central1-docker.pkg.dev/oss-fuzz/unsafe/skcms",
       "."
     ],
-    "dir": "oss-fuzz/projects/skcms"
+    "dir": "oss-fuzz/projects/skcms",
+    "id": "build-x86_64"
   },
   {
     "name": "gcr.io/oss-fuzz/skcms",

infra/experimental/chronos/README.md

@@ -1,23 +1,20 @@
-# Usage
-Under `OSS-Fuzz` root directory:
-```bash
-export PROJECT=libiec61850
-export FUZZ_TARGET=fuzz_mms_decode.c
-export FUZZING_LANGUAGE=c
+# Chronos: rebuilding OSS-Fuzz harnesses using cached builds
 
-infra/experimental/chronos/prepare-recompile "$PROJECT" "$FUZZ_TARGET" "$FUZZING_LANGUAGE"
-python infra/helper.py build_image "$PROJECT"
-# AddressSanitizer.
-docker run -ti --entrypoint="/bin/sh" --env SANITIZER="address" --name "${PROJECT}-origin-asan" "gcr.io/oss-fuzz/${PROJECT}" -c "compile && rm -rf /out/*"
-docker commit --change 'CMD ["compile"] --change 'ENTRYPOINT /bin/sh' "${PROJECT}-origin-asan" "gcr.io/oss-fuzz/${PROJECT}-ofg-cached-asan"
-docker run -ti --entrypoint="recompile" "gcr.io/oss-fuzz/${PROJECT}-ofg-cached-asan"
+## Usage locally
 
-# Coverage measurement.
-docker run -ti --entrypoint="/bin/sh" --env SANITIZER="coverage" --name "${PROJECT}-origin-cov" "gcr.io/oss-fuzz/${PROJECT}" -c "compile && rm -rf /out/*"
-docker commit --change 'CMD ["compile"] --change 'ENTRYPOINT /bin/sh' "${PROJECT}-origin-cov" "gcr.io/oss-fuzz/${PROJECT}-ofg-cached-cov"
-docker run -ti --entrypoint="recompile" "gcr.io/oss-fuzz/${PROJECT}-ofg-cached-cov"
-```
+**Example 1: htslib**
+
+From the OSS-Fuzz root
 
-# Assumptions
-1. Fuzzer: Chronos assumes `libFuzzer`. Other fuzzers are not well-supported, but may work by setting ENV `FUZZING_ENGINE` in project's `Dockerfile`.
-2. Sanitizer: Chronos assumes `AddressSanitizer`. Other sanitizers may work by adding setting ENV `SANITIZER` in project's `Dockerfile`.
+```sh
+$ RUN_ALL=1 ./infra/experimental/chronos/build_cache_local.sh htslib c
+...
+...
+Vanilla compile time:
+17
+Replay worked
+Replay compile time:
+2
+Ccache compile time: 
+9
+```