Merge pull request #376 from extremeshok/dev

7.2.4

.codeclimate.yml

@@ -21,3 +21,4 @@ ratings:
   paths: []
 exclude_paths:
   - .t/
+  - dev/

README.md

@@ -220,14 +220,21 @@ Usage: clamav-unofficial-sigs.sh   [OPTION] [PATH|FILE]
 
 ## Change Log
 
+### Version 7.2.4 (17 March 2021)
+
+* eXtremeSHOK.com Maintenance
+* Disabled winnow_malware.yara , duplicated in EMAIL_Cryptowall.yar and no longer maintained
+* Removed gtar requirement (--wildcards is the default)
+* Incremented the config to version 97
+
 ### Version 7.2.3 (17 March 2021)
 
 * eXtremeSHOK.com Maintenance
+* Whitelist support for yararules (whitelist signature tracking is disabled for yararules)
 * Disable JJencode.yar , due to excessive CPU usage
 * Disable scamnailer , discontinued
 * Fix working directory variable "urlhausy" to "urlhaus"
 * Update pfsense guide for 2.5
-* Whitelist support for yararules (whitelist signature tracking is disabled for yararules)
 * Fix missing tracker-tmp.txt
 * Thank you @perplexityjeff
 

clamav-unofficial-sigs.sh

@@ -1544,7 +1544,7 @@ EOF
 ################################################################################
 
 # Script Info
-script_version="7.2.3"
+script_version="7.2.4"
 script_version_date="2021-03-17"
 minimum_required_config_version="96"
 minimum_yara_clamav_version="0.100"
@@ -2020,14 +2020,14 @@ elif [[ "$rsync_bin" =~ "/" ]] ; then
 fi
 # tar_bin
 if [ -z "$tar_bin" ] ; then
-    # Detect support for tar or gtar
-    if [ "$(uname -s)" == "Darwin" ] || [ "$(uname -s)" == "OpenBSD" ] || [ "$(uname -s)" == "NetBSD" ] || [ "$(uname -s)" == "FreeBSD" ] ; then
-        tar_bin="$(command -v gtar 2> /dev/null)"
-    else
+    # # Detect support for tar or gtar
+    # if [ "$(uname -s)" == "Darwin" ] || [ "$(uname -s)" == "OpenBSD" ] || [ "$(uname -s)" == "NetBSD" ] || [ "$(uname -s)" == "FreeBSD" ] ; then
+    #     tar_bin="$(command -v gtar 2> /dev/null)"
+    # else
         tar_bin="$(command -v tar 2> /dev/null)"
-    fi
+    # fi
     if [ -z "$tar_bin" ] ; then
-        xshok_pretty_echo_and_log "ERROR: tar or gtar binary (tar_bin) not found"
+        xshok_pretty_echo_and_log "ERROR: tar binary (tar_bin) not found"
         exit 1
     fi
 elif [[ "$tar_bin" =~ "/" ]] ; then
@@ -3358,9 +3358,9 @@ if [ "$linuxmalwaredetect_enabled" == "yes" ] ; then
           if [ "$ret" -eq 0 ] ; then
                         # shellcheck disable=SC2035
             if [ "$enable_yararules" == "yes" ] ; then
-                $tar_bin --strip-components=1 --wildcards --overwrite -xzf "${work_dir_linuxmalwaredetect}/sigpack.tgz" --directory "${work_dir_linuxmalwaredetect}" */rfxn.*
+                $tar_bin --strip-components=1 --overwrite -xzf "${work_dir_linuxmalwaredetect}/sigpack.tgz" --directory "${work_dir_linuxmalwaredetect}" */rfxn.*
             else
-                $tar_bin --strip-components=1 --wildcards --exclude='*.yar' --exclude='*.yara' --overwrite -xzf "${work_dir_linuxmalwaredetect}/sigpack.tgz" --directory "${work_dir_linuxmalwaredetect}" */rfxn.*
+                $tar_bin --strip-components=1 --exclude='*.yar' --exclude='*.yara' --overwrite -xzf "${work_dir_linuxmalwaredetect}/sigpack.tgz" --directory "${work_dir_linuxmalwaredetect}" */rfxn.*
             fi
             for db_file in "${linuxmalwaredetect_dbs[@]}" ; do
               if [ "$loop" == "1" ] ; then

config/master.conf

@@ -252,7 +252,7 @@ winnow.complex.patterns.ldb|MEDIUM  # contain hand generated signatures for malw
 winnow_phish_complete.ndb|HIGH  # Phishing and other malicious urls and compromised hosts **DO NOT USE WITH winnow_phish_complete_url**
 ### OITC YARA Format rules
 ### Note: Yara signatures require ClamAV 0.100 or newer to work
-winnow_malware.yara|LOW  # detect spam
+winnow_malware.yara|DISABLED  # Duplicated in EMAIL_Cryptowall.yar and no longer maintaned
 
 ### MiscreantPunch http://malwarefor.me/about/
 ## MEDIUM
@@ -736,7 +736,7 @@ yararulesproject_url="https://raw.githubusercontent.com/Yara-Rules/rules/master"
 
 # ========================
 # DO NOT EDIT !
-config_version="96"
+config_version="97"
 
 ################################################################################
 #

config/os/os.macos.conf

@@ -20,7 +20,7 @@
 
 # Mac OS and OS X with clamav installed via homebrew
 
-# Requires gnu-sed (gsed) and gnu-tar (gtar)
+# Requires gnu-sed (gsed)
 
 # Follow the installation Instructions: see the guide in the guides folder
 

config/os/os.openbsd.conf

@@ -18,7 +18,7 @@
 # Rename to os.conf to enable this file
 ################################################################################
 
-# Requires gnu-sed (gsed) and gnu-tar (gtar)
+# Requires gnu-sed (gsed)
 
 # OpenBSD
 

dev/test_yara_rules.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+###################
+# This is property of eXtremeSHOK.com
+# You are free to use, modify and distribute, however you may not remove this notice.
+# Copyright (c) Adrian Jon Kriel :: [email protected]
+# License: BSD (Berkeley Software Distribution)
+##################
+# A small utility to check/verify Yara-Rules from https://github.com/Yara-Rules/rules
+#################
+export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/usr/local/musl/bin:$HOME/bin
+
+wget https://raw.githubusercontent.com/Yara-Rules/rules/master/index.yar -O /tmp/index.yar
+sed 's|include "./||g' /tmp/index.yar | sed 's|"||g' | sed -r ':a; s%(.*)/\*.*\*/%\1%; ta; /\/\*/ !b; N; ba' | sed '/^$/d' > /tmp/rules.yara
+
+echo "" > /tmp/empty-file
+
+while IFS= read -r line ; do
+  if [ -n "$line" ] ; then
+    # shellcheck disable=SC2086
+    sub_dir="${line/\/*}"
+    mkdir -p "/tmp/yara/${sub_dir}"
+
+    wget --quiet "https://raw.githubusercontent.com/Yara-Rules/rules/master/${line}" -O "/tmp/yara/${line}"
+
+    output="$(clamscan --quiet --no-summary --database="/tmp/yara/${line}" /tmp/empty-file 2>&1)"
+    ret="$?"
+
+    if [ -n "$output" ] || [ "$ret" != "0" ] ; then
+      echo "ERROR --- ${line} ---"
+    else
+      echo "--- ${line} ---"
+      #echo "$ret"
+      #echo "$output"
+    fi
+  fi
+done < "/tmp/rules.yara"
+
+
+# clamscan --database=antidebug_antivm.yar 2> scan.log
+#
+# egrep "yyerror()|yara" scan.log
+# check the errorlevel at this stage.
+
+# here is some testing code which identifies all rules in .yar file, checks for which ones are duplicated in rfxn.yara, then shows the name of the rules that are not duplicated.:
+# shellcheck disable=SC2062
+grep -ah "^rule " /var/lib/clamav/*.yar|cut -d: -f1 >/tmp/rules; while read -r RULE; do grep -qF "$RULE" /var/lib/clamav/rfxn.yara||echo "$RULE"; done</tmp/rules
+
+# And this does the same check but outputs the names of the .yar files where the non-duplicated rules are found:
+# shellcheck disable=SC2062
+grep -ah "^rule " /var/lib/clamav/*.yar|cut -d: -f1 >/tmp/rules; while read -r RULE; do grep -qF "$RULE" /var/lib/clamav/rfxn.yara||echo "$RULE"; done</tmp/rules|grep -Ff- /var/lib/clamav/*.yar

guides/macos.md

@@ -18,9 +18,9 @@ Tested on macOS Big Sur (OSX 11)
 ```
 
 
-# Step 2 Install dependencies : gtar (gnu-tar) sed (gnu-sed)
+# Step 2 Install dependencies : sed (gnu-sed)
 ```
-brew install gnu-tar gnu-sed
+brew install gnu-sed
 ```
 
 # Step 3 Install clamav