s22, installer

e-m-b-a · Dec 9, 2024 · 5de1988 · 5de1988
1 parent 9686f0a
commit 5de1988
Show file tree

Hide file tree

Showing 11 changed files with 46 additions and 46 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -40,6 +40,7 @@ services:
     environment:
         - USER
         - CONTAINER_NUMBER=1
+    # network_mode: host
     devices:
       - /dev/fuse:/dev/fuse:rwm
     cap_add:

diff --git a/helpers/helpers_emba_defaults.sh b/helpers/helpers_emba_defaults.sh
@@ -29,6 +29,7 @@ set_defaults() {
   export ARCH_CHECK=1
   export RTOS=1                 # Testing RTOS based OS - 1 -> no Linux / 0 -> Linux
   export BINARY_EXTENDED=0
+  export MAX_EXT_CHECK_BINS=20
   export CONTAINER_EXTRACT=0
   export DISABLE_DEEP=0
   export DEEP_EXT_DEPTH=4

diff --git a/helpers/helpers_emba_extractors.sh b/helpers/helpers_emba_extractors.sh
@@ -57,18 +57,3 @@ binwalker_matryoshka() {
   print_ln
 }
 
-binwalker_v2_matryoshka() {
-  local lFIRMWARE_PATH="${1:-}"
-  local lOUTPUT_DIR_BINWALK="${2:-}"
-
-  sub_module_title "Analyze binary firmware blob with binwalk"
-
-  print_output "[*] Extracting firmware to directory ${ORANGE}${lOUTPUT_DIR_BINWALK}${NC}"
-
-  if ! [[ -d "${lOUTPUT_DIR_BINWALK}" ]]; then
-    mkdir -p "${lOUTPUT_DIR_BINWALK}"
-  fi
-
-  timeout --preserve-status --signal SIGINT 300 "${BINWALK_BIN[@]}" --run-as=root --preserve-symlinks -e -M --dd='.*' -C "${lOUTPUT_DIR_BINWALK}" "${lFIRMWARE_PATH}" | tee -a "${LOG_FILE}" || true
-  print_ln
-}
diff --git a/helpers/helpers_emba_prepare.sh b/helpers/helpers_emba_prepare.sh
@@ -179,7 +179,7 @@ binary_architecture_threader() {
     # noreorder, pic, cpic, o32, mips32
     local lREADELF_H_ARR=()
 
-    mapfile -t lREADELF_H_ARR < <(readelf -h "${lBINARY}" 2>/dev/null)
+    mapfile -t lREADELF_H_ARR < <(readelf -h "${lBINARY}" 2>/dev/null || true)
 
     lD_FLAGS_CNT=$(printf -- '%s\n' "${lREADELF_H_ARR[@]}" | grep "Flags:" || true)
     lD_FLAGS_CNT="${lD_FLAGS_CNT// /}"

diff --git a/installer/I120_cwe_checker.sh b/installer/I120_cwe_checker.sh
@@ -27,6 +27,7 @@ I120_cwe_checker() {
     print_tool_info "gcc" 1
     print_tool_info "curl" 1
     print_tool_info "make" 1
+    print_tool_info "openjdk-17-jdk" 1
     # print_tool_info "rust-all" 1
     # print_tool_info "cargo" 1
 
@@ -52,26 +53,26 @@ I120_cwe_checker() {
         ## GHIDRA
 
         # Java SDK for ghidra
-        if [[ -d ./external/jdk ]] ; then rm -R ./external/jdk ; fi
-        curl -L https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.12%2B7/OpenJDK11U-jdk_x64_linux_hotspot_11.0.12_7.tar.gz -Sf -o external/jdk.tar.gz
-        mkdir external/jdk 2>/dev/null
-        tar -xzf external/jdk.tar.gz -C external/jdk --strip-components 1
-        rm external/jdk.tar.gz
+        # if [[ -d ./external/jdk ]] ; then rm -R ./external/jdk ; fi
+        # curl -L https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.12%2B7/OpenJDK11U-jdk_x64_linux_hotspot_11.0.12_7.tar.gz -Sf -o external/jdk.tar.gz
+        # mkdir external/jdk 2>/dev/null
+        # tar -xzf external/jdk.tar.gz -C external/jdk --strip-components 1
+        # rm external/jdk.tar.gz
 
         # Ghidra
         if [[ -d ./external/ghidra ]] ; then rm -R ./external/ghidra ; fi
-        # curl -L https://github.com/NationalSecurityAgency/ghidra/releases/download/Ghidra_10.3.1_build/ghidra_10.3.1_PUBLIC_20230614.zip -Sf -o external/ghidra.zip
-        curl -L https://github.com/NationalSecurityAgency/ghidra/releases/download/Ghidra_10.2.3_build/ghidra_10.2.3_PUBLIC_20230208.zip -Sf -o external/ghidra.zip
+        curl -L https://github.com/NationalSecurityAgency/ghidra/releases/download/Ghidra_10.3.1_build/ghidra_10.3.1_PUBLIC_20230614.zip -Sf -o external/ghidra.zip
+        # curl -L https://github.com/NationalSecurityAgency/ghidra/releases/download/Ghidra_10.2.3_build/ghidra_10.2.3_PUBLIC_20230208.zip -Sf -o external/ghidra.zip
         mkdir external/ghidra 2>/dev/null
         unzip -qo external/ghidra.zip -d external/ghidra
-        if [[ "${IN_DOCKER}" -eq 1 ]]; then
-          # sed -i s@JAVA_HOME_OVERRIDE=@JAVA_HOME_OVERRIDE=/external/jdk@g external/ghidra/ghidra_10.3.1_PUBLIC/support/launch.properties
-          sed -i s@JAVA_HOME_OVERRIDE=@JAVA_HOME_OVERRIDE=/external/jdk@g external/ghidra/ghidra_10.2.3_PUBLIC/support/launch.properties
-        else
-          # sed -i s@JAVA_HOME_OVERRIDE=@JAVA_HOME_OVERRIDE=external/jdk@g external/ghidra/ghidra_10.3.1_PUBLIC/support/launch.properties
-          sed -i s@JAVA_HOME_OVERRIDE=@JAVA_HOME_OVERRIDE=external/jdk@g external/ghidra/ghidra_10.2.3_PUBLIC/support/launch.properties
-        fi
-        rm external/ghidra.zip
+        # if [[ "${IN_DOCKER}" -eq 1 ]]; then
+        #   # sed -i s@JAVA_HOME_OVERRIDE=@JAVA_HOME_OVERRIDE=/external/jdk@g external/ghidra/ghidra_10.3.1_PUBLIC/support/launch.properties
+        #   sed -i s@JAVA_HOME_OVERRIDE=@JAVA_HOME_OVERRIDE=/external/jdk@g external/ghidra/ghidra_10.2.3_PUBLIC/support/launch.properties
+        # else
+        #   # sed -i s@JAVA_HOME_OVERRIDE=@JAVA_HOME_OVERRIDE=external/jdk@g external/ghidra/ghidra_10.3.1_PUBLIC/support/launch.properties
+        #   sed -i s@JAVA_HOME_OVERRIDE=@JAVA_HOME_OVERRIDE=external/jdk@g external/ghidra/ghidra_10.2.3_PUBLIC/support/launch.properties
+        # fi
+        # rm external/ghidra.zip
 
         # further Ghidra installation stuff:
         mkdir external/ghidra_scripts
@@ -84,13 +85,13 @@ I120_cwe_checker() {
           mkdir ./external/cwe_checker 2>/dev/null
           git clone https://github.com/EMBA-support-repos/cwe_checker.git external/cwe_checker
           cd external/cwe_checker || ( echo "Could not install EMBA component cwe_checker" && exit 1 )
-          # make all GHIDRA_PATH="${HOME_PATH}""/external/ghidra/ghidra_10.3.1_PUBLIC"
-          make all GHIDRA_PATH="${HOME_PATH}""/external/ghidra/ghidra_10.2.3_PUBLIC"
+          make all GHIDRA_PATH="${HOME_PATH}""/external/ghidra/ghidra_10.3.1_PUBLIC"
+          # make all GHIDRA_PATH="${HOME_PATH}""/external/ghidra/ghidra_10.2.3_PUBLIC"
           cd "${HOME_PATH}" || ( echo "Could not install EMBA component cwe_checker" && exit 1 )
 
           if [[ "${IN_DOCKER}" -eq 1 ]]; then
             # cp -pr "${HOME}""/.cargo/bin" "external/cwe_checker/bin"
-            echo '{"ghidra_path":"/external/ghidra/ghidra_10.2.3_PUBLIC"}' > "${HOME}"/.config/cwe_checker/ghidra.json
+            echo '{"ghidra_path":"/external/ghidra/ghidra_10.3.1_PUBLIC"}' > "${HOME}"/.config/cwe_checker/ghidra.json
 
             # save .config as we remount /root with tempfs -> now we can restore it in the module
             cp -pr "${HOME}"/.config ./external/cwe_checker/

diff --git a/installer/I13_disasm.sh b/installer/I13_disasm.sh
@@ -22,7 +22,7 @@ I13_disasm() {
   if [[ "${LIST_DEP}" -eq 1 ]] || [[ "${IN_DOCKER}" -eq 1 ]] || [[ "${DOCKER_SETUP}" -eq 0 ]] || [[ "${FULL}" -eq 1 ]]; then
 
     BINUTIL_VERSION_NAME="binutils-2.35.1"
-    CAPA_VERSION="7.3.0"
+    CAPA_VERSION="8.0.0"
 
     INSTALL_APP_LIST=()
 

diff --git a/modules/S16_ghidra_decompile_checks.sh b/modules/S16_ghidra_decompile_checks.sh
@@ -59,7 +59,7 @@ S16_ghidra_decompile_checks()
   fi
 
   for lBIN_TO_CHECK in "${lBINARIES_ARR[@]}"; do
-    if [[ -f "${BASE_LINUX_FILES}" && "${FULL_TEST}" -eq 0 ]]; then
+    if [[ -f "${BASE_LINUX_FILES}" ]]; then
       # if we have the base linux config file we only test non known Linux binaries
       # with this we do not waste too much time on open source Linux stuff
       lNAME=$(basename "${lBIN_TO_CHECK}" 2> /dev/null)
@@ -68,6 +68,11 @@ S16_ghidra_decompile_checks()
       fi
     fi
 
+    if ! [[ -f "${lBIN_TO_CHECK}" ]]; then
+      lBIN_TO_CHECK=$(grep "${lBIN_TO_CHECK}" "${P99_CSV_LOG}" | sort -u | head -1 || true)
+      print_output "[*] S16 - Testing ${lBIN_TO_CHECK}"
+    fi
+
     # ensure we have not tested this binary entry
     local lBIN_MD5=""
     lBIN_MD5="$(md5sum "${lBIN_TO_CHECK}" | awk '{print $1}')"
@@ -87,8 +92,8 @@ S16_ghidra_decompile_checks()
       ghidra_analyzer "${lBIN_TO_CHECK}"
     fi
 
-    # we stop checking after the first 20 binaries
-    if [[ "${#lBINS_CHECKED_ARR[@]}" -gt 20 ]] && [[ "${FULL_TEST}" -ne 1 ]]; then
+    # we stop checking after the first MAX_EXT_CHECK_BINS binaries
+    if [[ "${#lBINS_CHECKED_ARR[@]}" -gt "${MAX_EXT_CHECK_BINS}" ]] && [[ "${FULL_TEST}" -ne 1 ]]; then
       print_output "[*] 20 binaries already analysed - ending Ghidra binary analysis now." "no_log"
       print_output "[*] For complete analysis enable FULL_TEST." "no_log"
       break
@@ -147,7 +152,7 @@ ghidra_analyzer() {
   print_output "[*] Extracting decompiled code from binary ${ORANGE}${lNAME} / ${lBINARY}${NC} with Ghidra" "no_log"
   local lIDENTIFIER="${RANDOM}"
 
-  "${GHIDRA_PATH}"/support/analyzeHeadless "${LOG_PATH_MODULE}" "ghidra_${lNAME}_${lIDENTIFIER}" -import "${lBINARY}" -log "${LOG_PATH_MODULE}"/ghidra_"${lNAME}"_"${lIDENTIFIER}".txt -scriptPath "${EXT_DIR}"/ghidra_scripts -postScript Haruspex || print_error "[-] Error detected while Ghidra run for ${lNAME}"
+  "${GHIDRA_PATH}"/support/analyzeHeadless "${LOG_PATH_MODULE}" "ghidra_${lNAME}_${lIDENTIFIER}" -import "${lBINARY}" -log "${LOG_PATH_MODULE}"/ghidra_"${lNAME}"_"${lIDENTIFIER}".txt -scriptPath "${EXT_DIR}"/ghidra_scripts -postScript Haruspex || print_error "[-] Error detected while Ghidra Headless run for ${lNAME}"
 
   # Ghidra cleanup:
   if [[ -d "${LOG_PATH_MODULE}"/"ghidra_${lNAME}_${lIDENTIFIER}.rep" ]]; then

diff --git a/modules/S17_cwe_checker.sh b/modules/S17_cwe_checker.sh
@@ -91,12 +91,13 @@ cwe_check() {
     # usually binaries with strcpy or system calls are more interesting for further analysis
     # to keep analysis time low we only check these bins
     mapfile -t lBINARIES_ARR < <(grep -h "strcpy\|system" "${S13_CSV_LOG}" "${S14_CSV_LOG}" | sort -k 3 -t ';' -n -r | awk '{print $1}' || true)
+    # we usually get a path like /sbin/httpd which is not resolvable and needs to queried again in the P99_CSV_LOG later on
   else
     mapfile -t lBINARIES_ARR < <(grep -v "ASCII text\|Unicode text" "${P99_CSV_LOG}" | grep "ELF" | cut -d ';' -f1 || true)
   fi
 
   for lBIN_TO_CHECK in "${lBINARIES_ARR[@]}"; do
-    if [[ -f "${BASE_LINUX_FILES}" && "${FULL_TEST}" -eq 0 ]]; then
+    if [[ -f "${BASE_LINUX_FILES}" ]]; then
       # if we have the base linux config file we only test non known Linux binaries
       # with this we do not waste too much time on open source Linux stuff
       lNAME=$(basename "${lBIN_TO_CHECK}")
@@ -107,8 +108,11 @@ cwe_check() {
 
     # do not try to analyze kernel modules:
     [[ "${lBIN_TO_CHECK}" == *".ko" ]] && continue
+    if ! [[ -f "${lBIN_TO_CHECK}" ]]; then
+      lBIN_TO_CHECK=$(grep "${lBIN_TO_CHECK}" "${P99_CSV_LOG}" | sort -u | head -1 || true)
+      print_output "[*] S17 - Testing ${lBIN_TO_CHECK}"
+    fi
     # ensure we have not tested this binary entry
-    print_output "[*] S17 - Testing ${lBIN_TO_CHECK}"
     local lBIN_MD5=""
     lBIN_MD5="$(md5sum "${lBIN_TO_CHECK}" | awk '{print $1}')"
     if [[ "${lBINS_CHECKED_ARR[*]}" == *"${lBIN_MD5}"* ]]; then
@@ -132,9 +136,9 @@ cwe_check() {
     else
       cwe_checker_threaded "${lBIN_TO_CHECK}"
     fi
-    # we stop checking after the first 20 binaries
+    # we stop checking after the first MAX_EXT_CHECK_BINS binaries
     # usually these are non-linux binaries and ordered by the usage of system/strcpy legacy usages
-    if [[ "${#lBINS_CHECKED_ARR[@]}" -gt 20 ]] && [[ "${FULL_TEST}" -ne 1 ]]; then
+    if [[ "${#lBINS_CHECKED_ARR[@]}" -gt "${MAX_EXT_CHECK_BINS}" ]] && [[ "${FULL_TEST}" -ne 1 ]]; then
       print_output "[*] 20 binaries already analysed - ending Ghidra binary analysis now." "no_log"
       print_output "[*] For complete analysis enable FULL_TEST." "no_log"
       break

diff --git a/modules/S22_php_check.sh b/modules/S22_php_check.sh
@@ -101,7 +101,7 @@ s22_vuln_check_semgrep() {
     # highlight security findings in semgrep log:
     sed -i -r "s/.*external\.semgrep-rules\.php\.lang\.security.*/\x1b[32m&\x1b[0m/" "${lPHP_SEMGREP_LOG/\.log/\.pretty\.log}"
 
-    mapfile -t lSEMG_SOURCES_ARR < <(grep -E -o -e "classname=\".*\"" -e "file=.*\"" -e "line=\"[0-9]+" "${lPHP_SEMGREP_LOG/\.log/\.pretty\.log}" | sed -z 's/"\n/\ /g' | sort -u || true)
+    mapfile -t lSEMG_SOURCES_ARR < <(grep -E -o -e "testcase name=\".*\"" -e "file=.*\"" -e "line=\"[0-9]+" "${lPHP_SEMGREP_LOG/\.log/\.pretty\.log}" | sed -z 's/"\n/\ /g' | sort -u || true)
 
     for lSEMG_SOURCE_NOTE in "${lSEMG_SOURCES_ARR[@]}"; do
       local lSEMG_ISSUE_NAME=""
@@ -113,8 +113,8 @@ s22_vuln_check_semgrep() {
 
       ! [[ -d "${LOG_PATH_MODULE}"/semgrep_sources/ ]] && mkdir "${LOG_PATH_MODULE}"/semgrep_sources/
 
-      lSEMG_ISSUE_NAME=$(echo "${lSEMG_SOURCE_NOTE}" | tr ' ' '\n' | grep "^classname=")
-      lSEMG_ISSUE_NAME="${lSEMG_ISSUE_NAME/classname=\"/}"
+      lSEMG_ISSUE_NAME=$(echo "${lSEMG_SOURCE_NOTE}" | tr ' ' '\n' | grep "^name=")
+      lSEMG_ISSUE_NAME="${lSEMG_ISSUE_NAME/name=\"/}"
 
       lSEMG_SOURCE_FILE=$(echo "${lSEMG_SOURCE_NOTE}" | tr ' ' '\n' | grep "^file=")
       lSEMG_SOURCE_FILE="${lSEMG_SOURCE_FILE/file=\"/}"

diff --git a/scan-profiles/default-scan.emba b/scan-profiles/default-scan.emba
@@ -28,6 +28,8 @@ export YARA=0
 # extended binary tests are now only testing non-linux binaries. With this mechanism we can enable
 # it in the default profile.
 export BINARY_EXTENDED=1
+# we test only the first 10 binaries with cwe_checker and extended semgrep rules
+export MAX_EXT_CHECK_BINS=10
 
 # the following modules are long running modules which are disabled in the default profile
 export MODULE_BLACKLIST+=( "S10_binaries_basic_check" "S15_radare_decompile_checks" "S99_grepit" "S110_yara_check" )

diff --git a/scan-profiles/full-scan.emba b/scan-profiles/full-scan.emba
@@ -19,6 +19,7 @@ export THREADED=1
 export SHORT_PATH=1
 export HTML=1
 export BINARY_EXTENDED=1
+export MAX_EXT_CHECK_BINS=100
 export QEMULATION=1
 export FULL_EMULATION=1
 export FULL_TEST=1