Merge pull request #1341 from m-1-k-3/multiple_updates

Multiple little fixes / updated base image

docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   # nosemgrep
   emba:
-    image: embeddedanalyzer/emba:1.4.2g
+    image: embeddedanalyzer/emba:1.4.2h
     container_name: emba
     read_only: true
     # all pre-checker mount modules need privileged mode
@@ -50,7 +50,7 @@ services:
         soft: 0
 
   emba_quest:
-    image: embeddedanalyzer/emba:1.4.2g
+    image: embeddedanalyzer/emba:1.4.2h
     container_name: emba_quest
     read_only: true
     tmpfs:

helpers/helpers_emba_defaults.sh

@@ -175,6 +175,7 @@ set_log_paths() {
   export S13_LOG="${LOG_DIR}/s13_weak_func_check.txt"
   export S13_CSV_LOG="${CSV_DIR}/s13_weak_func_check.csv"
   export S14_LOG="${LOG_DIR}/s14_weak_func_radare_check.txt"
+  export S14_CSV_LOG="${CSV_DIR}/s14_weak_func_radare_check.csv"
   export S16_LOG="${LOG_DIR}/s16_ghidra_decompile_checks.txt"
   export S17_LOG="${LOG_DIR}/s17_cwe_checker.txt"
   export S17_CSV_LOG="${CSV_DIR}/s17_apk_check.csv"

installer/IP00_extractors.sh

@@ -27,6 +27,9 @@ IP00_extractors(){
     print_git_info "payload_dumper" "EMBA-support-repos/payload_dumper" "Android OTA payload.bin extractor"
     print_git_info "smcbmc" "EMBA-support-repos/smcbmc" "Supermicro BMC firmware image decryptor"
     print_git_info "dji-firmware-tools" "EMBA-support-repos/dji-firmware-tools" "Tools for extracting, modding and re-packaging firmwares of DJI multirotor drones."
+    print_tool_info "python3-pycryptodome" 1
+    # sometimes the python pip installation is needed - probably this will be solved in the future
+    # probably it depends on the venv?!?
     print_pip_info "pycryptodome"
     # ubireader:
     # print_tool_info "python3-lzo" 1
@@ -58,6 +61,7 @@ IP00_extractors(){
         pip_install "protobuf"
         pip_install "bsdiff4"
         pip_install "python-lzo>=1.14"
+        pip_install "pycryptodome"
 
         if ! [[ -d external/payload_dumper ]]; then
           git clone https://github.com/EMBA-support-repos/payload_dumper.git external/payload_dumper

installer/IP99_binwalk_default.sh

@@ -40,6 +40,9 @@ IP99_binwalk_default() {
     print_tool_info "cabextract" 1
     print_tool_info "util-linux" 1
     print_tool_info "python3-matplotlib" 1
+    # sometimes the python pip installation is needed - probably this will be solved in the future
+    # probably it depends on the venv?!?
+    print_pip_info "matplotlib"
 
     # tools only available on Kali Linux:
     if [[ "${OTHER_OS}" -eq 0 ]] && [[ "${UBUNTU_OS}" -eq 0 ]]; then
@@ -91,6 +94,7 @@ IP99_binwalk_default() {
       y|Y )
         apt-get install "${INSTALL_APP_LIST[@]}" -y --no-install-recommends
         pip_install "setuptools"
+        pip_install "matplotlib"
 
         git clone https://github.com/EMBA-support-repos/binwalk_ospg.git external/binwalk
         cd external/binwalk || ( echo "Could not install EMBA component binwalk" && exit 1 )

modules/F20_vul_aggregator.sh

@@ -315,7 +315,9 @@ aggregate_versions() {
       print_output "[*] Software inventory aggregated:"
       for VERSION in "${VERSIONS_AGGREGATED[@]}"; do
         # ensure our set anchor is based on the binary name and is limited to 20 characters:
-        local ANCHOR="${VERSION/:*/}"
+        local ANCHOR=""
+        ANCHOR=$(echo "${VERSION}" | cut -d ':' -f3-4)
+        ANCHOR="${ANCHOR//:/_}"
         ANCHOR="cve_${ANCHOR:0:20}"
         print_output "[+] Found Version details (${ORANGE}aggregated${GREEN}): ""${ORANGE}${VERSION}${NC}"
         write_link "f20#${ANCHOR}"
@@ -1082,7 +1084,10 @@ cve_extractor() {
 
   local BIN_LOG="${LOG_PATH_MODULE}/cve_details_${BINARY}_${VERSION}.log"
   write_log "[*] Vulnerability details for ${ORANGE}${BINARY}${NC} / version ${ORANGE}${VERSION}${NC} / source ${ORANGE}${VSOURCE}${NC}:" "${BIN_LOG}"
-  write_anchor "cve_${BINARY:0:20}" "${BIN_LOG}"
+  local ANCHOR=""
+  ANCHOR="${BINARY}_${VERSION}"
+  ANCHOR="cve_${ANCHOR:0:20}"
+  write_anchor "${ANCHOR}" "${BIN_LOG}"
   if [[ "${EXPLOIT_COUNTER_VERSION}" -gt 0 ]]; then
     write_log "" "${BIN_LOG}"
     grep -v "Statistics" "${LOG_PATH_MODULE}"/cve_sum/"${AGG_LOG_FILE}" >> "${BIN_LOG}" || true

modules/P18_BMC_decryptor.sh

@@ -50,7 +50,7 @@ bmc_extractor() {
   "${EXT_DIR}"/smcbmc/smcbmc.py "${lBMC_FILE_PATH_}" "${lEXTRACTION_FILE_}" || print_error "[-] BMC decryption failed for ${lBMC_FILE_PATH_}"
 
   print_ln
-  if [[ -f "${lEXTRACTION_FILE_}" ]]; then
+  if [[ -s "${lEXTRACTION_FILE_}" ]]; then
     export FIRMWARE_PATH="${lEXTRACTION_FILE_}"
     print_output "[+] Extracted BMC encrypted firmware file to ${ORANGE}${FIRMWARE_PATH}${NC}"
     backup_var "FIRMWARE_PATH" "${FIRMWARE_PATH}"

modules/S16_ghidra_decompile_checks.sh

@@ -46,14 +46,16 @@ S16_ghidra_decompile_checks()
   local BINS_CHECKED_ARR=()
 
   if [[ "${FULL_TEST}" -ne 1 ]]; then
-    # we only need to wait if we are not using the full_scan profile
+    # we need to wait in default mode for the results of S13 and S14
     module_wait "S13_weak_func_check"
+    module_wait "S14_weak_func_radare_check"
   fi
-  if [[ -f "${S13_CSV_LOG}" ]]; then
-    local BINARIES=()
+
+  if [[ -f "${S13_CSV_LOG}" ]] || [[ -f "${S14_CSV_LOG}" ]]; then
     # usually binaries with strcpy or system calls are more interesting for further analysis
     # to keep analysis time low we only check these bins
-    mapfile -t BINARIES < <(grep "strcpy\|system" "${S13_CSV_LOG}" | sort -k 3 -t ';' -n -r | awk '{print $1}' || true)
+    local BINARIES=()
+    mapfile -t BINARIES < <(grep -h "strcpy\|system" "${S13_CSV_LOG}" "${S14_CSV_LOG}" | sort -k 3 -t ';' -n -r | awk '{print $1}' || true)
   fi
 
   for BINARY in "${BINARIES[@]}"; do

modules/S17_cwe_checker.sh

@@ -32,10 +32,10 @@ S17_cwe_checker()
     local lCWE_CNT_=0
     local lTESTED_BINS=0
 
-    # [[ "${IN_DOCKER}" -eq 1 ]] && cwe_container_prepare
     if [[ "${FULL_TEST}" -ne 1 ]]; then
-      # we only need to wait if we are not using the full_scan profile
+      # we need to wait in default mode for the results of S13 and S14
       module_wait "S13_weak_func_check"
+      module_wait "S14_weak_func_radare_check"
     fi
 
     cwe_check
@@ -87,11 +87,11 @@ cwe_check() {
   local lNAME=""
   local lBINS_CHECKED_ARR=()
 
-  if [[ -f "${S13_CSV_LOG}" ]]; then
-    local BINARIES=()
+  if [[ -f "${S13_CSV_LOG}" ]] || [[ -f "${S14_CSV_LOG}" ]]; then
     # usually binaries with strcpy or system calls are more interesting for further analysis
     # to keep analysis time low we only check these bins
-    mapfile -t BINARIES < <(grep "strcpy\|system" "${S13_CSV_LOG}" | sort -k 3 -t ';' -n -r | awk '{print $1}' || true)
+    local BINARIES=()
+    mapfile -t BINARIES < <(grep -h "strcpy\|system" "${S13_CSV_LOG}" "${S14_CSV_LOG}" | sort -k 3 -t ';' -n -r | awk '{print $1}' || true)
   fi
 
   for lBINARY in "${BINARIES[@]}" ; do

modules/S18_capa_checker.sh

@@ -35,11 +35,11 @@ S18_capa_checker() {
     # we only need to wait if we are not using the full_scan profile
     module_wait "S13_weak_func_check"
   fi
-  if [[ -s "${S13_CSV_LOG}" ]]; then
-    local BINARIES=()
+  if [[ -f "${S13_CSV_LOG}" ]] || [[ -f "${S14_CSV_LOG}" ]]; then
     # usually binaries with strcpy or system calls are more interesting for further analysis
     # to keep analysis time low we only check these bins
-    mapfile -t BINARIES < <(grep "strcpy\|system" "${S13_CSV_LOG}" | sort -k 3 -t ';' -n -r | awk '{print $1}' || true)
+    local BINARIES=()
+    mapfile -t BINARIES < <(grep -h "strcpy\|system" "${S13_CSV_LOG}" "${S14_CSV_LOG}" | sort -k 3 -t ';' -n -r | awk '{print $1}' || true)
   fi
 
   local lBINARY=""
@@ -82,12 +82,12 @@ S18_capa_checker() {
 
         # in normal operation we stop checking after the first 20 binaries
         # if FULL_TEST is activated we are testing all binaries -> this takes a long time
-        lBINS_CHECKED_CNT=$(wc -l "${TMP_DIR}"/s18_checked.tmp 2>/dev/null || true)
-        if [[ "${lBINS_CHECKED_CNT/\ *}" -gt 20 ]] && [[ "${FULL_TEST}" -ne 1 ]]; then
-          print_output "[*] 20 binaries already analysed - ending capa binary analysis now." "no_log"
-          print_output "[*] For complete analysis enable FULL_TEST." "no_log"
-          break 2
-        fi
+        # lBINS_CHECKED_CNT=$(wc -l "${TMP_DIR}"/s18_checked.tmp 2>/dev/null || true)
+        # if [[ "${lBINS_CHECKED_CNT/\ *}" -gt 40 ]] && [[ "${FULL_TEST}" -ne 1 ]]; then
+        #  print_output "[*] 20 binaries already analysed - ending capa binary analysis now." "no_log"
+        #  print_output "[*] For complete analysis enable FULL_TEST." "no_log"
+        #  break 2
+        # fi
       else
         print_output "[-] Binary behavior testing with capa for $(print_path "${lBIN_TO_CHECK}") not possible ... unsupported architecture" "no_log"
       fi