From a5c3854e66f1c8b88edfaab024b354da77936549 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Thu, 18 Mar 2021 05:52:57 +0200 Subject: [PATCH 1/6] small utility to test yararules --- dev/test_yara_rules.sh | 48 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 dev/test_yara_rules.sh diff --git a/dev/test_yara_rules.sh b/dev/test_yara_rules.sh new file mode 100644 index 00000000..28546cb4 --- /dev/null +++ b/dev/test_yara_rules.sh @@ -0,0 +1,48 @@ +#!/bin/sh +################### +# This is property of eXtremeSHOK.com +# You are free to use, modify and distribute, however you may not remove this notice. +# Copyright (c) Adrian Jon Kriel :: admin@extremeshok.com +# License: BSD (Berkeley Software Distribution) +################## +# A small utility to check/verify Yara-Rules from https://github.com/Yara-Rules/rules +################# +export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/usr/local/musl/bin:$HOME/bin + +wget https://raw.githubusercontent.com/Yara-Rules/rules/master/index.yar -O /tmp/index.yar +sed 's|include "./||g' /tmp/index.yar | sed 's|"||g' | sed -r ':a; s%(.*)/\*.*\*/%\1%; ta; /\/\*/ !b; N; ba' | sed '/^$/d' > /tmp/rules.yara + +echo "" > /tmp/empty-file + +while IFS= read -r line ; do + if [ ! -z "$line" ] ; then + # shellcheck disable=SC2086 + sub_dir="${line/\/*}" + mkdir -p "/tmp/yara/${sub_dir}" + + wget --quiet "https://raw.githubusercontent.com/Yara-Rules/rules/master/${line}" -O "/tmp/yara/${line}" + + output="$(clamscan --quiet --no-summary --database="/tmp/yara/${line}" /tmp/empty-file 2>&1)" + ret="$?" + + if [ ! -z "$output" ] || [ "$ret" != "0" ] ; then + echo "ERROR --- ${line} ---" + else + echo "--- ${line} ---" + #echo "$ret" + #echo "$output" + fi + fi +done < "/tmp/rules.yara" + + +# clamscan --database=antidebug_antivm.yar 2> scan.log +# +# egrep "yyerror()|yara" scan.log +# check the errorlevel at this stage. + +# here is some testing code which identifies all rules in .yar file, checks for which ones are duplicated in rfxn.yara, then shows the name of the rules that are not duplicated.: +grep -ah "^rule " /var/lib/clamav/*.yar|cut -d: -f1 >/tmp/rules; while read RULE; do grep -qF "$RULE" /var/lib/clamav/rfxn.yara||echo $RULE; done/tmp/rules; while read RULE; do grep -qF "$RULE" /var/lib/clamav/rfxn.yara||echo $RULE; done Date: Thu, 18 Mar 2021 06:03:17 +0200 Subject: [PATCH 2/6] Fixes #203 --- README.md | 8 +++++++- config/master.conf | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e28949f6..937e1a64 100644 --- a/README.md +++ b/README.md @@ -220,14 +220,20 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] ## Change Log +### Version 7.2.4 (17 March 2021) + +* eXtremeSHOK.com Maintenance +* Disabled winnow_malware.yara , duplicated in EMAIL_Cryptowall.yar and no longer maintained +* Incremented the config to version 97 + ### Version 7.2.3 (17 March 2021) * eXtremeSHOK.com Maintenance +* Whitelist support for yararules (whitelist signature tracking is disabled for yararules) * Disable JJencode.yar , due to excessive CPU usage * Disable scamnailer , discontinued * Fix working directory variable "urlhausy" to "urlhaus" * Update pfsense guide for 2.5 -* Whitelist support for yararules (whitelist signature tracking is disabled for yararules) * Fix missing tracker-tmp.txt * Thank you @perplexityjeff diff --git a/config/master.conf b/config/master.conf index e7d6f13f..66f1c97b 100644 --- a/config/master.conf +++ b/config/master.conf @@ -252,7 +252,7 @@ winnow.complex.patterns.ldb|MEDIUM # contain hand generated signatures for malw winnow_phish_complete.ndb|HIGH # Phishing and other malicious urls and compromised hosts **DO NOT USE WITH winnow_phish_complete_url** ### OITC YARA Format rules ### Note: Yara signatures require ClamAV 0.100 or newer to work -winnow_malware.yara|LOW # detect spam +winnow_malware.yara|DISABLED # Duplicated in EMAIL_Cryptowall.yar and no longer maintaned ### MiscreantPunch http://malwarefor.me/about/ ## MEDIUM @@ -736,7 +736,7 @@ yararulesproject_url="https://raw.githubusercontent.com/Yara-Rules/rules/master" # ======================== # DO NOT EDIT ! -config_version="96" +config_version="97" ################################################################################ # From 60aef8579c90565b4919f837fbaa40f49319d08a Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Thu, 18 Mar 2021 06:13:44 +0200 Subject: [PATCH 3/6] Removed gtar requirement #303 --- README.md | 1 + clamav-unofficial-sigs.sh | 16 ++++++++-------- config/os/os.macos.conf | 2 +- config/os/os.openbsd.conf | 2 +- guides/macos.md | 4 ++-- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 937e1a64..d11ea6f8 100644 --- a/README.md +++ b/README.md @@ -224,6 +224,7 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] * eXtremeSHOK.com Maintenance * Disabled winnow_malware.yara , duplicated in EMAIL_Cryptowall.yar and no longer maintained +* Removed gtar requirement (--wildcards is the default) * Incremented the config to version 97 ### Version 7.2.3 (17 March 2021) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index e51184d2..ba0595d8 100755 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -2020,14 +2020,14 @@ elif [[ "$rsync_bin" =~ "/" ]] ; then fi # tar_bin if [ -z "$tar_bin" ] ; then - # Detect support for tar or gtar - if [ "$(uname -s)" == "Darwin" ] || [ "$(uname -s)" == "OpenBSD" ] || [ "$(uname -s)" == "NetBSD" ] || [ "$(uname -s)" == "FreeBSD" ] ; then - tar_bin="$(command -v gtar 2> /dev/null)" - else + # # Detect support for tar or gtar + # if [ "$(uname -s)" == "Darwin" ] || [ "$(uname -s)" == "OpenBSD" ] || [ "$(uname -s)" == "NetBSD" ] || [ "$(uname -s)" == "FreeBSD" ] ; then + # tar_bin="$(command -v gtar 2> /dev/null)" + # else tar_bin="$(command -v tar 2> /dev/null)" - fi + # fi if [ -z "$tar_bin" ] ; then - xshok_pretty_echo_and_log "ERROR: tar or gtar binary (tar_bin) not found" + xshok_pretty_echo_and_log "ERROR: tar binary (tar_bin) not found" exit 1 fi elif [[ "$tar_bin" =~ "/" ]] ; then @@ -3358,9 +3358,9 @@ if [ "$linuxmalwaredetect_enabled" == "yes" ] ; then if [ "$ret" -eq 0 ] ; then # shellcheck disable=SC2035 if [ "$enable_yararules" == "yes" ] ; then - $tar_bin --strip-components=1 --wildcards --overwrite -xzf "${work_dir_linuxmalwaredetect}/sigpack.tgz" --directory "${work_dir_linuxmalwaredetect}" */rfxn.* + $tar_bin --strip-components=1 --overwrite -xzf "${work_dir_linuxmalwaredetect}/sigpack.tgz" --directory "${work_dir_linuxmalwaredetect}" */rfxn.* else - $tar_bin --strip-components=1 --wildcards --exclude='*.yar' --exclude='*.yara' --overwrite -xzf "${work_dir_linuxmalwaredetect}/sigpack.tgz" --directory "${work_dir_linuxmalwaredetect}" */rfxn.* + $tar_bin --strip-components=1 --exclude='*.yar' --exclude='*.yara' --overwrite -xzf "${work_dir_linuxmalwaredetect}/sigpack.tgz" --directory "${work_dir_linuxmalwaredetect}" */rfxn.* fi for db_file in "${linuxmalwaredetect_dbs[@]}" ; do if [ "$loop" == "1" ] ; then diff --git a/config/os/os.macos.conf b/config/os/os.macos.conf index 8abd5b7d..13fbdf4b 100644 --- a/config/os/os.macos.conf +++ b/config/os/os.macos.conf @@ -20,7 +20,7 @@ # Mac OS and OS X with clamav installed via homebrew -# Requires gnu-sed (gsed) and gnu-tar (gtar) +# Requires gnu-sed (gsed) # Follow the installation Instructions: see the guide in the guides folder diff --git a/config/os/os.openbsd.conf b/config/os/os.openbsd.conf index 8cbad62b..a6f015df 100644 --- a/config/os/os.openbsd.conf +++ b/config/os/os.openbsd.conf @@ -18,7 +18,7 @@ # Rename to os.conf to enable this file ################################################################################ -# Requires gnu-sed (gsed) and gnu-tar (gtar) +# Requires gnu-sed (gsed) # OpenBSD diff --git a/guides/macos.md b/guides/macos.md index 05b236d5..6741744f 100644 --- a/guides/macos.md +++ b/guides/macos.md @@ -18,9 +18,9 @@ Tested on macOS Big Sur (OSX 11) ``` -# Step 2 Install dependencies : gtar (gnu-tar) sed (gnu-sed) +# Step 2 Install dependencies : sed (gnu-sed) ``` -brew install gnu-tar gnu-sed +brew install gnu-sed ``` # Step 3 Install clamav From 414624bca811945df877b24160cd053fea1c3d7b Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Thu, 18 Mar 2021 06:18:15 +0200 Subject: [PATCH 4/6] bump version --- clamav-unofficial-sigs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index ba0595d8..a81f0765 100755 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -1544,7 +1544,7 @@ EOF ################################################################################ # Script Info -script_version="7.2.3" +script_version="7.2.4" script_version_date="2021-03-17" minimum_required_config_version="96" minimum_yara_clamav_version="0.100" From a8489ae9eaf3b7dad2b01bc49024e845a9098004 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Thu, 18 Mar 2021 06:25:10 +0200 Subject: [PATCH 5/6] happy codeclimate --- dev/test_yara_rules.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/dev/test_yara_rules.sh b/dev/test_yara_rules.sh index 28546cb4..875e859f 100644 --- a/dev/test_yara_rules.sh +++ b/dev/test_yara_rules.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash ################### # This is property of eXtremeSHOK.com # You are free to use, modify and distribute, however you may not remove this notice. @@ -15,7 +15,7 @@ sed 's|include "./||g' /tmp/index.yar | sed 's|"||g' | sed -r ':a; s%(.*)/\*.*\* echo "" > /tmp/empty-file while IFS= read -r line ; do - if [ ! -z "$line" ] ; then + if [ -n "$line" ] ; then # shellcheck disable=SC2086 sub_dir="${line/\/*}" mkdir -p "/tmp/yara/${sub_dir}" @@ -25,7 +25,7 @@ while IFS= read -r line ; do output="$(clamscan --quiet --no-summary --database="/tmp/yara/${line}" /tmp/empty-file 2>&1)" ret="$?" - if [ ! -z "$output" ] || [ "$ret" != "0" ] ; then + if [ -n "$output" ] || [ "$ret" != "0" ] ; then echo "ERROR --- ${line} ---" else echo "--- ${line} ---" @@ -42,7 +42,9 @@ done < "/tmp/rules.yara" # check the errorlevel at this stage. # here is some testing code which identifies all rules in .yar file, checks for which ones are duplicated in rfxn.yara, then shows the name of the rules that are not duplicated.: -grep -ah "^rule " /var/lib/clamav/*.yar|cut -d: -f1 >/tmp/rules; while read RULE; do grep -qF "$RULE" /var/lib/clamav/rfxn.yara||echo $RULE; done/tmp/rules; while read -r RULE; do grep -qF "$RULE" /var/lib/clamav/rfxn.yara||echo "$RULE"; done/tmp/rules; while read RULE; do grep -qF "$RULE" /var/lib/clamav/rfxn.yara||echo $RULE; done/tmp/rules; while read -r RULE; do grep -qF "$RULE" /var/lib/clamav/rfxn.yara||echo "$RULE"; done Date: Thu, 18 Mar 2021 06:30:36 +0200 Subject: [PATCH 6/6] ignore dev/ directory --- .codeclimate.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.codeclimate.yml b/.codeclimate.yml index 9531be25..0edb9fcc 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -21,3 +21,4 @@ ratings: paths: [] exclude_paths: - .t/ + - dev/