From 1fb25c8a2074caeada445491432f74e720a198d6 Mon Sep 17 00:00:00 2001 From: Quentin JEROME Date: Thu, 5 Nov 2020 22:18:56 +0100 Subject: [PATCH] =?UTF-8?q?Addressed=20issue:=C2=A0#10?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- engine/engine.go | 106 +++++++++++++++++++++++++++++++++++++++++- engine/engine_test.go | 93 +++++++++++++++++++++++++++++++++++- rules/rules.go | 5 ++ 3 files changed, 202 insertions(+), 2 deletions(-) diff --git a/engine/engine.go b/engine/engine.go index cac5e4d..11c754d 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -126,6 +126,7 @@ type Engine struct { trace bool dumpRaw bool showAttck bool + filter bool // tells that we should apply Filter rules // Used to mark the traces and not duplicate those markedTraces datastructs.SyncedSet containers *rules.ContainerDB @@ -453,7 +454,7 @@ func (e *Engine) AddTraceRules(ruleList ...*rules.CompiledRule) { } } -//Match checks if there is a match in any rule of the engine +//Match (deprecated) checks if there is a match in any rule of the engine func (e *Engine) Match(event *evtx.GoEvtxMap) (names []string, criticality int) { var matched bool @@ -461,10 +462,12 @@ func (e *Engine) Match(event *evtx.GoEvtxMap) (names []string, criticality int) names = make([]string, 0) attcks := make([]rules.Attack, 0) markedAttcks := datastructs.NewSyncedSet() + filtered := true e.RLock() for _, r := range e.rules { if r.Match(event) { + filtered = filtered && r.Filter matched = true names = append(names, r.Name) // Updating ATT&CK information @@ -502,6 +505,107 @@ func (e *Engine) Match(event *evtx.GoEvtxMap) (names []string, criticality int) // Unlock so that we can update engine e.RUnlock() + // tells that the only matches are filters + if filtered { + // we keep original event unmodified + return + } + + // We can update with the traces since we released the lock + e.AddTraceRules(traces...) + + // Bound criticality + criticality = globals.Bound(criticality) + // Update event with signature information + genInfo := map[string]interface{}{ + "Signature": names, + "Criticality": criticality} + + // Update ATT&CK information if needed + if e.showAttck && len(attcks) > 0 { + genInfo["ATTACK"] = attcks + } + + event.Set(&geneInfoPath, genInfo) + // Update engine's statistics + e.Lock() + e.Stats.Scanned++ + if matched { + e.Stats.Positives++ + } + e.Unlock() + return +} + +// MatchOrFilter checks if there is a match in any rule of the engine. The only difference with Match function is that +// it also return a flag indicating if the event is filtered. +func (e *Engine) MatchOrFilter(event *evtx.GoEvtxMap) (names []string, criticality int, filtered bool) { + var matched bool + + // return values + names = make([]string, 0) + filtered = true + + // initialized variables + traces := make([]*rules.CompiledRule, 0) + attcks := make([]rules.Attack, 0) + markedAttcks := datastructs.NewSyncedSet() + + e.RLock() + for _, r := range e.rules { + if r.Match(event) { + filtered = filtered && r.Filter + matched = true + names = append(names, r.Name) + + // Do not need to go further if it is a filter rule + if r.Filter { + continue + } + + // Updating ATT&CK information + for _, attck := range r.Attck { + if !markedAttcks.Contains(attck.ID) { + attcks = append(attcks, attck) + markedAttcks.Add(attck.ID) + } + } + + criticality += r.Criticality + // If we decide to trace the other events matching the rules + if e.trace { + for i, tr := range r.Traces { + value, err := event.GetString(tr.Path()) + // If we find the appropriate element in the event we matched + if err == nil { + // Hashing the trace + h := tr.HashWithValue(value) + if !e.markedTraces.Contains(h) { + // We add the hash of the current trace not to recompile it again + e.markedTraces.Add(h) + // We compile the trace into a rule and append it to the list of traces + if tRule, err := tr.Compile(r, value); err == nil { + traces = append(traces, tRule) + } else { + log.Errorf("Failed to compile trace rule i=%d for \"%s\" ", i, r.Name) + } + } + } + } + } + } + } + // Unlock so that we can update engine + e.RUnlock() + + // it is an filtered event if at least one rule matched and filtered flag is true + filtered = len(names) > 0 && filtered + // tells that the only matches are filters + if filtered { + // we keep original event unmodified + return + } + // We can update with the traces since we released the lock e.AddTraceRules(traces...) diff --git a/engine/engine_test.go b/engine/engine_test.go index 1037ceb..b45df68 100644 --- a/engine/engine_test.go +++ b/engine/engine_test.go @@ -138,7 +138,7 @@ func prettyJSON(i interface{}) string { } func openEvtx(path string) *evtx.File { - f, err := evtx.New(path) + f, err := evtx.OpenDirty(path) if err != nil { panic(err) } @@ -484,6 +484,97 @@ func TestContainer(t *testing.T) { } } +func TestFiltered(t *testing.T) { + /* + "CommandLine": "C:\\Windows\\system32\\devicecensus.exe", + "CurrentDirectory": "C:\\Windows\\system32\\", + "Hashes": "SHA1=65894B0162897F2A6BB8D2EB13684BF2B451FDEE,MD5=83514D9AAF0E168944B6D3C01110C393,SHA256=03324E67244312360FF089CF61175DEF2031BE513457BB527AE0ABF925E72319,IMPHASH=D9EA1DE97F43E8F8608832D8E83DA2CF", + "Image": "C:\\Windows\\System32\\DeviceCensus.exe", + "IntegrityLevel": "System", + "LogonGuid": "B2796A13-618F-5881-0000-0020E7030000", + "LogonId": "0x000003e7", + "ParentCommandLine": "C:\\Windows\\system32\\svchost.exe -k netsvcs", + "ParentImage": "C:\\Windows\\System32\\svchost.exe", + "ParentProcessGuid": "B2796A13-6191-5881-0000-00100FD80000", + "ParentProcessId": "828", + "ProcessGuid": "B2796A13-E4BA-5880-0000-00102BC01100", + "ProcessId": "3516", + "TerminalSessionId": "0", + "User": "NT AUTHORITY\\SYSTEM", + "UtcTime": "2017-01-19 16:09:30.252" + */ + rule := `{ + "Name": "ProcessCreate", + "Meta": { + "Channels": ["Microsoft-Windows-Sysmon/Operational"], + "EventIDs": [1], + "Filter": true + }, + "Matches": [], + "Condition": "" + } + ` + e := NewEngine(false) + err := e.LoadReader(NewSeekBuffer([]byte(rule))) + if err != nil { + t.Fail() + t.Log(err) + } + t.Logf("Successfuly loaded %d rules", e.Count()) + // The match should fail + if _, _, filtered := e.MatchOrFilter(&event); filtered { + t.Log("Event correctly filtered") + t.Logf("%s", prettyJSON(event)) + } else { + t.Fail() + } +} + +func TestNotFiltered(t *testing.T) { + /* + "CommandLine": "C:\\Windows\\system32\\devicecensus.exe", + "CurrentDirectory": "C:\\Windows\\system32\\", + "Hashes": "SHA1=65894B0162897F2A6BB8D2EB13684BF2B451FDEE,MD5=83514D9AAF0E168944B6D3C01110C393,SHA256=03324E67244312360FF089CF61175DEF2031BE513457BB527AE0ABF925E72319,IMPHASH=D9EA1DE97F43E8F8608832D8E83DA2CF", + "Image": "C:\\Windows\\System32\\DeviceCensus.exe", + "IntegrityLevel": "System", + "LogonGuid": "B2796A13-618F-5881-0000-0020E7030000", + "LogonId": "0x000003e7", + "ParentCommandLine": "C:\\Windows\\system32\\svchost.exe -k netsvcs", + "ParentImage": "C:\\Windows\\System32\\svchost.exe", + "ParentProcessGuid": "B2796A13-6191-5881-0000-00100FD80000", + "ParentProcessId": "828", + "ProcessGuid": "B2796A13-E4BA-5880-0000-00102BC01100", + "ProcessId": "3516", + "TerminalSessionId": "0", + "User": "NT AUTHORITY\\SYSTEM", + "UtcTime": "2017-01-19 16:09:30.252" + */ + rule := `{ + "Name": "ProcessCreate", + "Meta": { + "Channels": ["Microsoft-Windows-Sysmon/Operational"], + "EventIDs": [2], + "Filter": true + }, + "Matches": [], + "Condition": "" + } + ` + e := NewEngine(false) + err := e.LoadReader(NewSeekBuffer([]byte(rule))) + if err != nil { + t.Fail() + t.Log(err) + } + t.Logf("Successfuly loaded %d rules", e.Count()) + // The match should fail + if _, _, filtered := e.MatchOrFilter(&event); !filtered { + t.Log("Event not filtered") + } else { + t.Fail() + } +} + func TestLoadDirectory(t *testing.T) { e := NewEngine(false) err := e.LoadDirectory("./") diff --git a/rules/rules.go b/rules/rules.go index f8a50ca..adda3fd 100644 --- a/rules/rules.go +++ b/rules/rules.go @@ -29,6 +29,7 @@ type CompiledRule struct { AtomMap datastructs.SyncedMap Traces []*Trace Disabled bool // Way to deal with no container issue + Filter bool // whether it is a Filter rule or not Conditions *ConditionElement containers *ContainerDB // ATT&CK information @@ -145,6 +146,7 @@ type MetaSection struct { Attack []Attack `json:"ATTACK,omitempty"` Criticality int Disable bool + Filter bool } //Rule is a JSON parsable rule @@ -216,6 +218,9 @@ func (jr *Rule) Compile(containers *ContainerDB) (*CompiledRule, error) { rule.Channels.Add(s) } + // Set Filter member + rule.Filter = jr.Meta.Filter + // Parses and Initializes the Traces for i, st := range jr.Meta.Traces { var tr *Trace