Improve clang-tidy check

industrial_ci/src/tests/merge_fixes.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2022, Robert Haschke
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import yaml
+import sys
+
+
+def key(item):
+   name = item.get("DiagnosticName")
+   msg = item.get("DiagnosticMessage")
+   file = msg.get("FilePath")
+   offset = msg.get("FileOffset")
+   return name, file, offset
+
+def merge_fixes(files):
+   """Merge all fixes files into mergefile"""
+   # The fixes suggested by clang-tidy >= 4.0.0 are given under
+   # the top level key 'Diagnostics' in the output yaml files
+   mergefile = files[0]
+   mergekey = "Diagnostics"
+   merged = []
+   seen = set()  # efficiently remember fixes already inserted
+
+   def have(x):
+      k = key(x)
+      return k in seen or seen.add(k)
+
+   def unique(seq):
+      return [x for x in seq if not have(x)]
+
+   for file in files:
+      try:
+         with open(file, 'r') as inp:
+            content = yaml.safe_load(inp)
+            if not content:
+               continue  # Skip empty files.
+            merged.extend(unique(content.get(mergekey, [])))
+      except FileNotFoundError:
+         pass
+
+   with open(mergefile, 'w') as out:
+      if merged:
+         # Assemble output dict with MainSourceFile=''.
+         output = {'MainSourceFile': '', mergekey: merged}
+         yaml.safe_dump(output, out)
+
+
+if __name__ == "__main__":
+   merge_fixes(sys.argv[1:])

industrial_ci/src/tests/source_tests.sh

@@ -63,32 +63,16 @@ function run_clang_tidy {
         ici_error "CLANG_TIDY_JOBS=$CLANG_TIDY_JOBS is invalid."
     fi
 
-    rm -rf "$db".{command,warn,error}
-    cat > "$db.command" << EOF
-#!/bin/bash
-num_non_file_args=\$1; shift
-args=("\${@:1:\$num_non_file_args}")
-files=("\${@:\$((num_non_file_args+1))}")
-fixes=\$(mktemp)
-rm -f /tmp/clang_tidy_output.\$\$
-for f in "\${files[@]}" ; do
-  ( cd \$(dirname \$f); clang-tidy "-export-fixes=\$fixes" "-header-filter=$regex" "-p=$build" "\${args[@]}" \$f &>> /tmp/clang_tidy_output.\$\$ 2>&1 || { touch "$db.error"; } )
-  if [ -s "\$fixes" ]; then touch "$db.warn"; fi
-done
-rm -rf "\$fixes"
-EOF
-    chmod +x "$db.command"
-
+    local err=0
     ici_log "run clang-tidy for ${#files[@]}/$num_all_files file(s) in $max_jobs process(es)."
+    set -o pipefail
+    printf "%s\0" "${files[@]}" | xargs --null run-clang-tidy "-j$max_jobs" "-header-filter=\"$regex\"" "-p=$build" "$@" 2>&1 | tee "$db.tidy.log" || err=$?
+    set +o pipefail
 
-    printf "%s\0" "${files[@]}" | xargs --null -P "$max_jobs" -n "$(( (${#files[@]} + max_jobs-1) / max_jobs))" "$db.command" "$#" "$@"
-    cat /tmp/clang_tidy_output.* | grep -vP "^([0-9]+ warnings generated|Use .* to display errors from system headers as well)\.$" || true
-    rm -rf /tmp/clang_tidy_output.*
-
-    if [ -f "$db.error" ]; then
+    if [ "$err" -ne "0" ]; then
        _run_clang_tidy_errors+=("$name")
-       ici_time_end "${ANSI_RED}"
-    elif [ -f "$db.warn" ]; then
+       ici_time_end "${ANSI_RED}" "$err"
+    elif grep -q "warning: " "$db.tidy.log"; then
         _run_clang_tidy_warnings+=("$name")
         ici_time_end "${ANSI_YELLOW}"
     else
@@ -110,10 +94,30 @@ function run_clang_tidy_check {
 
     ici_hook "before_clang_tidy_checks"
 
+    # replace -export-fixes <filename> with temporary file
+    local fixes_final=""
+    local fixes_tmp
+    local num_args=${#clang_tidy_args[@]}
+    fixes_tmp=$(mktemp)
+    for (( i=0; i<num_args; i++ )); do
+        if [ "${clang_tidy_args[i]}" == "-export-fixes" ]; then
+            fixes_final="${clang_tidy_args[i+1]}"
+            clang_tidy_args[i+1]="$fixes_tmp"
+        fi
+    done
+
+    # run clang-tidy checks on all build folders in target_ws
     while read -r db; do
         run_clang_tidy "$target_ws/src" warnings errors "$db" "${clang_tidy_args[@]}"
+        if [ -n "${fixes_final}" ]; then
+            "${ICI_SRC_PATH}/tests/merge_fixes.py" "$fixes_final" "$fixes_tmp"
+        fi
     done < <(find "$target_ws/build" -mindepth 2 -name compile_commands.json)  # -mindepth 2, because colcon puts a compile_commands.json into the build folder
 
+    if [ -n "${fixes_final}" ]; then
+        # translate file names in fixes file
+        sed -i "s#$target_ws/src/$TARGET_REPO_NAME/#$TARGET_REPO_PATH/#g" "${fixes_final}"
+    fi
     ici_hook "after_clang_tidy_checks"
 
     if [ "${#warnings[@]}" -gt "0" ]; then