Merge pull request #1404 from m-1-k-3/helpers_refactor

Unhandled  files in SBOM

config/bin_version_strings.cfg

@@ -287,7 +287,7 @@ inno_setup;;unknown;"^Inno\ Setup\ Messages\ \([0-9]\.[0-9]\.[0-9]\)\ \(u\)";"se
 inotifywatch;;unknown;"^inotifywatch\ [0-9](\.[0-9]+)+?$";"sed -r 's/inotifywatch\ ([0-9](\.[0-9]+)+?)$/:inotify:inotify-tools:\1/'";
 inotifywait;;unknown;"^inotifywait\ [0-9](\.[0-9]+)+?$";"sed -r 's/inotifywait\ ([0-9](\.[0-9]+)+?)$/:inotify:inotify-tools:\1/'";
 intel_trusted_device_setup;;unknown;"^Intel\(R\)\ Trusted\ Device\ Setup\ Extension\ Version\ [0-9]+(\.[0-9]+)+?$";"sed -r 's/Intel\(R\)\ Trusted\ Device\ Setup\ Extension\ Version\ ([0-9]+(\.[0-9]+)+?)/:intel:trusted_device_setup:\1/'";
-nichestack;;proprietary;^Interniche\ Stack\ v[0-9](\.[0-9]+)+$";"sed -r 's/Interniche\ Stack\ v([0-9](\.[0-9]+)+?)$/:hcc-embedded:nichestack:\1/'";
+nichestack;;proprietary;"^Interniche\ Stack\ v[0-9](\.[0-9]+)+$";"sed -r 's/Interniche\ Stack\ v([0-9](\.[0-9]+)+?)$/:hcc-embedded:nichestack:\1/'";
 io-control;;unknown;"FUSE\ library\ version:\ [0-9](\.[0-9]+)+?";"sed -r 's/FUSE\ library\ version:\ ([0-9](\.[0-9]+)+?).*/:fuse:fuse:\1/'";
 iotgoat;;MIT;"^iotgoat\ v[0-9]\.[0-9]$";"sed -r 's/iotgoat\ v([0-9](\.[0-9]+)+?)$/::iotgoat:\1/'";
 iperf;;unknown;"iperf\ version\ [0-9](\.[0-9]+)+?";"sed -r 's/iperf\ version\ ([0-9](\.[0-9]+)+?).*/::iperf:\1/'";
@@ -548,7 +548,7 @@ openswan;;GPL-2.0-only;"^Openswan\ [0-9](\.[0-9]+)+?$";"sed -r 's/Openswan\ ([0-
 openswan;;GPL-2.0-only;"^Linux\ Openswan\ [0-9](\.[0-9]+)+?$";"sed -r 's/Linux\ Openswan\ ([0-9](\.[0-9]+)+?)$/::openswan:\1/'";
 openvpn;;GPL-2.0-only;"^OpenVPN\ [0-9](\.[0-9]+)+?\ ";"sed -r 's/OpenVPN\ ([0-9](\.[0-9]+)+?)\ .*/:openvpn:openvpn:\1/'";
 # for future use / not yet used
-openwrt;;gpl;(OpenWrt)\ ([0-9]+\.[0-9]+\.[0-9])\ (r[0-9]+\-)([a-z0-9]+).*";sed -r 's/(OpenWrt)\ ([0-9]+\.[0-9]+\.[0-9])\ (r[0-9]+\-)([a-z0-9]+).*/:openwrt:openwrt:\2/'";
+openwrt;;gpl;"(OpenWrt)\ ([0-9]+\.[0-9]+\.[0-9])\ (r[0-9]+\-)([a-z0-9]+).*";sed -r 's/(OpenWrt)\ ([0-9]+\.[0-9]+\.[0-9])\ (r[0-9]+\-)([a-z0-9]+).*/:openwrt:openwrt:\2/'";
 opkg;;unknown;"opkg\ version\ [0-9](\.[0-9]+)+?";"sed -r 's/opkg\ version\ ([0-9](\.[0-9]+)+?)/::opkg:\1/'";
 ospf6d;;unknown;"^ospf6d\ version\ [0-9](\.[0-9]+)+$";"sed -r 's/ospf6d\ version\ ([0-9](\.[0-9]+)+?)$/::ospf6d:\1/'";
 overlord;;unknown;"^overlord\ [0-9](\.[0-9]+)+?$";"sed -r 's/overlord\ ([0-9](\.[0-9]+)+?)$/::overlord:\1/'";

helpers/helpers_emba_defaults.sh

@@ -159,6 +159,7 @@ set_defaults() {
   # we limit the maximal file log of our SBOM -> change this in the scanning profile
   export SBOM_MAX_FILE_LOG=200
   export SBOM_MINIMAL=0
+  export SBOM_UNTRACKED_FILES=1
 
   # we can enable/disable the s08 submodules with the following array configuration
   # -> just comment the submodule that should not be used
@@ -182,7 +183,7 @@ set_defaults() {
 }
 
 set_log_paths() {
-  export SBOM_LOG_PATH="${LOG_DIR}/SBOM/"
+  export SBOM_LOG_PATH="${LOG_DIR}/SBOM"
   export P02_CSV_LOG="${CSV_DIR}/p02_firmware_bin_file_check.csv"
   export P99_CSV_LOG="${CSV_DIR}/p99_prepare_analyzer.csv"
   export P55_LOG="${LOG_DIR}/p55_unblob_extractor.txt"
@@ -206,7 +207,6 @@ set_log_paths() {
   export S16_LOG="${LOG_DIR}/s16_ghidra_decompile_checks.txt"
   export S17_LOG="${LOG_DIR}/s17_cwe_checker.txt"
   export S17_CSV_LOG="${CSV_DIR}/s17_apk_check.csv"
-  export S24_CSV_LOG="${CSV_DIR}/s24_kernel_bin_identifier.csv"
   export S25_CSV_LOG="${CSV_DIR}/s25_kernel_check.csv"
   export S20_LOG="${LOG_DIR}/s20_shell_check.txt"
   export S21_LOG="${LOG_DIR}/s21_python_check.txt"

helpers/helpers_emba_sbom_helpers.sh

@@ -166,6 +166,18 @@ build_sbom_json_hashes_arr() {
       fi
     done
   fi
+
+  if [[ "${lPACKAGING_SYSTEM}" != "unhandled_file" && -d "${SBOM_LOG_PATH}" ]]; then
+    # Finally, we check if there is another "unhandled_file_*.json" with the same hash. If we find such a file we can remove it now
+    mapfile -t lDUP_CHECK_FILE_ARR < <(grep -lr '"alg":"SHA-512","content":"'"${lSHA512_CHECKSUM}" "${SBOM_LOG_PATH}"/unhandled_file_*.json 2>/dev/null || true)
+    for lDUP_CHECK_FILE in "${lDUP_CHECK_FILE_ARR[@]}"; do
+      print_output "[*] Duplicate unhandled_file sbom entry detected for ${lAPP_NAME} - ${lDUP_CHECK_FILE}" "no_log"
+      if ! grep -q "${lDUP_CHECK_FILE}" "${SBOM_LOG_PATH}"/duplicates_to_delete.txt 2>/dev/null; then
+        echo "${lDUP_CHECK_FILE}" >> "${SBOM_LOG_PATH}"/duplicates_to_delete.txt
+      fi
+    done
+  fi
+
   return 0
 
   # lhashes=$(jo -p -a "${HASHES_ARR[@]}")

modules/F15_cyclonedx_sbom.sh

@@ -81,6 +81,15 @@ F15_cyclonedx_sbom() {
     lTOOL_COMP_ARR+=( version="${lSBOM_TOOL_VERS}" )
     lTOOL_COMP_ARR+=( description="EMBA firmware analyzer - https://github.com/e-m-b-a/emba" )
 
+    # the following removes the duplicate untracked files that are handled from an other SBOM entry
+    if [[ -s "${SBOM_LOG_PATH}"/duplicates_to_delete.txt ]]; then
+      local lDUP_DEL=""
+      print_output "[*] Deleting duplicates" "no_log"
+      while read -r lDUP_DEL; do
+        rm -f "${lDUP_DEL}" || true
+      done < "${SBOM_LOG_PATH}"/duplicates_to_delete.txt
+    fi
+
     # Firmeware details for the SBOM
     local lFW_COMPONENT_DATA_ARR=()
     lFW_COMPONENT_DATA_ARR+=( name="${lFW_PATH}" )
@@ -96,13 +105,23 @@ F15_cyclonedx_sbom() {
     [[ -v HASHES_ARR ]] && lFW_COMPONENT_DATA_ARR+=( "hashes=$(jo -a "${HASHES_ARR[@]}")" )
 
     # build the component array for final sbom build:
-    mapfile -t lCOMP_FILES_ARR < <(find "${SBOM_LOG_PATH}" -maxdepth 1 -type f -name "*.json" | sort -u)
+    mapfile -t lCOMP_FILES_ARR < <(find "${SBOM_LOG_PATH}" -maxdepth 1 -type f -name "*.json" -not -name "unhandled_file_*" | sort -u)
+    if [[ "${SBOM_UNTRACKED_FILES}" -gt 0 ]]; then
+      mapfile -t lCOMP_FILES_ARR_UNHANDLED < <(find "${SBOM_LOG_PATH}" -maxdepth 1 -type f -name "unhandled_file_*.json" | sort -u)
+     lCOMP_FILES_ARR+=("${lCOMP_FILES_ARR_UNHANDLED[@]}")
+    fi
 
-    # as we could have so many components that everything goes b00m we need to build the
-    # components json now manually:
+    # as we can have so many components that everything goes b00m we need to build the
+    # components json manually:
     echo -n "[" > "${SBOM_LOG_PATH}/sbom_components_tmp.json"
     for lCOMP_FILE_ID in "${!lCOMP_FILES_ARR[@]}"; do
       lCOMP_FILE="${lCOMP_FILES_ARR["${lCOMP_FILE_ID}"]}"
+
+      if [[ "${SBOM_UNTRACKED_FILES:-0}" -ne 1 ]] && [[ "${lCOMP_FILE}" == *"unhandled_file_"* ]]; then
+        # if we do not include unhandled_file entries we can skipe them here
+        continue
+      fi
+
       if [[ -s "${lCOMP_FILE}" ]]; then
         cat "${lCOMP_FILE}" >> "${SBOM_LOG_PATH}/sbom_components_tmp.json"
       else
@@ -153,6 +172,7 @@ F15_cyclonedx_sbom() {
           "${lFW_COMPONENT_DATA_ARR[@]}")")" \
       components=:"${lSBOM_LOG_FILE}_components.json" \
       dependencies=:"${lSBOM_LOG_FILE}_dependencies.json" \
+      vulnerabilities="[]" \
       > "${lSBOM_LOG_FILE}.json" || print_error "[-] SBOM builder error!"
 
     # I am sure there is a much cleaner way but for now I am stuck and don't get it in a different way :(

modules/F50_base_aggregator.sh

@@ -1087,11 +1087,12 @@ print_os() {
   lSYSTEM=$(echo "${lSYSTEM}" | tr -dc '[:print:]')
 
   if [[ ${VERIFIED} -eq 1 ]]; then
+    print_output "[+] Operating system detected (""${ORANGE}""verified${GREEN}): ${ORANGE}${lSYSTEM}${NC}"
     if [[ "${VERIFIED_S03}" -eq 1 ]]; then
-      print_output "[+] Operating system detected (""${ORANGE}""verified${GREEN}): ${ORANGE}${lSYSTEM}${NC}"
       write_link "s03"
+    elif [[ -f "${S24_LOG}" ]]; then
+      write_link "s24"
     else
-      print_output "[+] Operating system detected (""${ORANGE}""verified${GREEN}): ${ORANGE}${lSYSTEM}${NC}"
       write_link "s25"
     fi
     write_csv_log "os_verified" "${lSYSTEM}" "NA" "NA" "NA" "NA" "NA" "NA" "NA"

modules/P50_binwalk_extractor.sh

@@ -67,14 +67,14 @@ P50_binwalk_extractor() {
       lFILES_EXT_BW=$(find "${OUTPUT_DIR_BINWALK}" -xdev -type f | wc -l )
       lUNIQUE_FILES_BW=$(find "${OUTPUT_DIR_BINWALK}" "${EXCL_FIND[@]}" -xdev -type f -exec md5sum {} \; | sort -u -k1,1 | cut -d\  -f3 | wc -l )
       lDIRS_EXT_BW=$(find "${OUTPUT_DIR_BINWALK}" -xdev -type d | wc -l )
-      lBINS_BW=$(find "${OUTPUT_DIR_BINWALK}" "${EXCL_FIND[@]}" -xdev -type f -exec file {} \; | grep -c "ELF" || true)
+      # lBINS_BW=$(find "${OUTPUT_DIR_BINWALK}" "${EXCL_FIND[@]}" -xdev -type f -exec file {} \; | grep -c "ELF" || true)
     fi
 
     if [[ "${lBINS_BW}" -gt 0 ]] || [[ "${lFILES_EXT_BW}" -gt 0 ]]; then
       sub_module_title "Firmware extraction details"
       print_output "[*] ${ORANGE}Binwalk${NC} results:"
       print_output "[*] Found ${ORANGE}${lFILES_EXT_BW}${NC} files (${ORANGE}${lUNIQUE_FILES_BW}${NC} unique files) and ${ORANGE}${lDIRS_EXT_BW}${NC} directories at all."
-      print_output "[*] Found ${ORANGE}${lBINS_BW}${NC} binaries."
+      # print_output "[*] Found ${ORANGE}${lBINS_BW}${NC} binaries."
       print_output "[*] Additionally the Linux path counter is ${ORANGE}${LINUX_PATH_COUNTER_BINWALK}${NC}."
       print_ln
       tree -sh "${OUTPUT_DIR_BINWALK}" | tee -a "${LOG_FILE}"

modules/P55_unblob_extractor.sh

@@ -92,14 +92,14 @@ P55_unblob_extractor() {
       lFILES_EXT_UB=$(find "${OUTPUT_DIR_UNBLOB}" -xdev -type f | wc -l)
       lUNIQUE_FILES_UB=$(find "${OUTPUT_DIR_UNBLOB}" "${EXCL_FIND[@]}" -xdev -type f -print0|xargs -r -0 -P 16 -I % sh -c 'md5sum "%" 2>/dev/null' | sort -u -k1,1 | cut -d\  -f3 | wc -l || true)
       lDIRS_EXT_UB=$(find "${OUTPUT_DIR_UNBLOB}" -xdev -type d | wc -l )
-      lBINS_UB=$(find "${OUTPUT_DIR_UNBLOB}" "${EXCL_FIND[@]}" -xdev -type f -print0|xargs -r -0 -P 16 -I % sh -c 'file %' 2>/dev/null | grep -c "ELF" || true )
+      # lBINS_UB=$(find "${OUTPUT_DIR_UNBLOB}" "${EXCL_FIND[@]}" -xdev -type f -print0|xargs -r -0 -P 16 -I % sh -c 'file %' 2>/dev/null | grep -c "ELF" || true )
     fi
 
     if [[ "${lBINS_UB}" -gt 0 ]] || [[ "${lFILES_EXT_UB}" -gt 0 ]]; then
       sub_module_title "Firmware extraction details"
       print_output "[*] ${ORANGE}Unblob${NC} results:"
       print_output "[*] Found ${ORANGE}${lFILES_EXT_UB}${NC} files (${ORANGE}${lUNIQUE_FILES_UB}${NC} unique files) and ${ORANGE}${lDIRS_EXT_UB}${NC} directories at all."
-      print_output "[*] Found ${ORANGE}${lBINS_UB}${NC} binaries."
+      # print_output "[*] Found ${ORANGE}${lBINS_UB}${NC} binaries."
       print_output "[*] Additionally the Linux path counter is ${ORANGE}${LINUX_PATH_COUNTER_UNBLOB}${NC}."
       print_ln
       tree -sh "${OUTPUT_DIR_UNBLOB}" | tee -a "${LOG_FILE}"

modules/P60_deep_extractor.sh

@@ -57,7 +57,7 @@ P60_deep_extractor() {
   sub_module_title "Extraction results"
 
   lUNIQUE_FILES=$(find "${FIRMWARE_PATH_CP}" "${EXCL_FIND[@]}" -xdev -type f -print0|xargs -r -0 -P 16 -I % sh -c 'md5sum "%" || true' 2>/dev/null | sort -u -k1,1 | cut -d\  -f3 | wc -l )
-  lBINS=$(find "${FIRMWARE_PATH_CP}" "${EXCL_FIND[@]}" -xdev -type f -print0|xargs -r -0 -P 16 -I % sh -c 'file "%" | grep -c "ELF"' || true)
+  # lBINS=$(find "${FIRMWARE_PATH_CP}" "${EXCL_FIND[@]}" -xdev -type f -print0|xargs -r -0 -P 16 -I % sh -c 'file "%" | grep -c "ELF"' || true)
   lFILES_EXT=$(find "${FIRMWARE_PATH_CP}" -xdev -type f | wc -l )
 
   if [[ "${lBINS}" -gt 0 || "${lUNIQUE_FILES}" -gt 0 ]]; then
@@ -66,7 +66,7 @@ P60_deep_extractor() {
     linux_basic_identification_helper "${FIRMWARE_PATH_CP}"
     print_ln
     print_output "[*] Found ${ORANGE}${lFILES_EXT}${NC} files (${ORANGE}${lUNIQUE_FILES}${NC} unique files) and ${ORANGE}${lDIRS_EXT}${NC} directories at all."
-    print_output "[*] Found ${ORANGE}${lBINS}${NC} binaries."
+    # print_output "[*] Found ${ORANGE}${lBINS}${NC} binaries."
     print_output "[*] Additionally the Linux path counter is ${ORANGE}${LINUX_PATH_COUNTER}${NC}."
 
     tree -csh "${FIRMWARE_PATH_CP}" | tee -a "${LOG_FILE}"