From b2af2e183ce84bca1920a1386379cb5ccc096042 Mon Sep 17 00:00:00 2001 From: caffix Date: Thu, 17 Oct 2024 20:13:55 -0400 Subject: [PATCH] new relations are now inserted into the database --- engine/plugins/dns/reverse.go | 43 +++++++++++++++-- engine/plugins/horizontals/contact.go | 1 + engine/plugins/horizontals/fqdn.go | 67 ++++++++++++--------------- engine/plugins/horizontals/plugin.go | 51 ++++++++++++++++++++ engine/sessions/scope/assoc.go | 4 +- 5 files changed, 122 insertions(+), 44 deletions(-) diff --git a/engine/plugins/dns/reverse.go b/engine/plugins/dns/reverse.go index c9f2a5d54..c611dd92f 100644 --- a/engine/plugins/dns/reverse.go +++ b/engine/plugins/dns/reverse.go @@ -59,11 +59,10 @@ func (d *dnsReverse) check(e *et.Event) error { return errors.New("failed to obtain the plugin source information") } - ptr := d.createPTRAlias(e, reverse, src) + ptr := d.createPTRAlias(e, reverse, e.Asset, src) if ptr == nil { return nil } - e.Session.Cache().SetAsset(ptr) since, err := support.TTLStartTime(e.Session.Config(), "IPAddress", "FQDN", d.plugin.name) if err != nil { @@ -171,7 +170,7 @@ func (d *dnsReverse) store(e *et.Event, ptr, src *dbt.Asset, rr []*resolve.Extra return rev } -func (d *dnsReverse) createPTRAlias(e *et.Event, name string, datasrc *dbt.Asset) *dbt.Asset { +func (d *dnsReverse) createPTRAlias(e *et.Event, name string, ip, datasrc *dbt.Asset) *dbt.Asset { done := make(chan *dbt.Asset, 1) defer close(done) @@ -181,14 +180,48 @@ func (d *dnsReverse) createPTRAlias(e *et.Event, name string, datasrc *dbt.Asset return } - ptr, err := e.Session.DB().Create(nil, "", &domain.FQDN{Name: name}) + ptr, err := e.Session.DB().Create(ip, "ptr_record", &domain.FQDN{Name: name}) if err == nil && ptr != nil { _, _ = e.Session.DB().Link(ptr, "source", datasrc) } done <- ptr }) - return <-done + ptr := <-done + if ptr == nil { + return ptr + } + + _ = e.Dispatcher.DispatchEvent(&et.Event{ + Name: ptr.Asset.Key(), + Asset: ptr, + Session: e.Session, + }) + + now := time.Now() + e.Session.Cache().SetRelation(&dbt.Relation{ + Type: "source", + CreatedAt: now, + LastSeen: now, + FromAsset: ptr, + ToAsset: datasrc, + }) + + if a, hit := e.Session.Cache().GetAsset(&domain.FQDN{Name: ptr.Asset.Key()}); hit && ptr != nil { + e.Session.Cache().SetRelation(&dbt.Relation{ + Type: "ptr_record", + CreatedAt: now, + LastSeen: now, + FromAsset: ip, + ToAsset: a, + }) + + e.Session.Log().Info("relationship discovered", "from", + a.Asset.Key(), "relation", "ptr_record", "to", ip.Asset.Key(), + slog.Group("plugin", "name", d.plugin.name, "handler", d.name)) + } + + return ptr } func (d *dnsReverse) process(e *et.Event, rev []*relRev, src *dbt.Asset) { diff --git a/engine/plugins/horizontals/contact.go b/engine/plugins/horizontals/contact.go index 9bd442ddb..5de4e0ecb 100644 --- a/engine/plugins/horizontals/contact.go +++ b/engine/plugins/horizontals/contact.go @@ -66,6 +66,7 @@ func (h *horContact) check(e *et.Event) error { if len(assets) > 0 { h.plugin.process(e, assets, src) + h.plugin.addAssociatedRelationship(e, assocs) } } return nil diff --git a/engine/plugins/horizontals/fqdn.go b/engine/plugins/horizontals/fqdn.go index a361ddd3f..0c8fbe7ed 100644 --- a/engine/plugins/horizontals/fqdn.go +++ b/engine/plugins/horizontals/fqdn.go @@ -7,9 +7,7 @@ package horizontals import ( "errors" "fmt" - "net/netip" - "github.com/coredns/coredns/plugin/pkg/dnsutil" "github.com/owasp-amass/amass/v4/engine/plugins/support" "github.com/owasp-amass/amass/v4/engine/sessions/scope" et "github.com/owasp-amass/amass/v4/engine/types" @@ -17,7 +15,6 @@ import ( oam "github.com/owasp-amass/open-asset-model" "github.com/owasp-amass/open-asset-model/domain" oamnet "github.com/owasp-amass/open-asset-model/network" - "github.com/owasp-amass/resolve" "golang.org/x/net/publicsuffix" ) @@ -60,7 +57,7 @@ func (h *horfqdn) check(e *et.Event) error { } if hit && len(rels) > 0 { - h.checkPTR(e, fqdn.Name, rels, src) + h.checkPTR(e, rels, e.Asset, src) return nil } @@ -84,48 +81,44 @@ func (h *horfqdn) check(e *et.Event) error { if len(assets) > 0 { h.plugin.process(e, assets, src) + h.plugin.addAssociatedRelationship(e, assocs) } } return nil } -func (h *horfqdn) checkPTR(e *et.Event, name string, rels []*dbt.Relation, src *dbt.Asset) { - if ipstr := dnsutil.ExtractAddressFromReverse(name + "."); ipstr != "" { - ipstr = resolve.RemoveLastDot(ipstr) - - addr, err := netip.ParseAddr(ipstr) - if err != nil { - return - } - - ip := &oamnet.IPAddress{Address: addr, Type: "IPv4"} - if ip.Address.Is6() { - ip.Type = "IPv6" - } +func (h *horfqdn) checkPTR(e *et.Event, rels []*dbt.Relation, fqdn, src *dbt.Asset) { + if rs, hit := e.Session.Cache().GetIncomingRelations(fqdn, "ptr_record"); hit && len(rs) > 0 { + for _, r := range rs { + ip, ok := r.FromAsset.Asset.(*oamnet.IPAddress) + if !ok { + continue + } - var inscope bool - _, conf := e.Session.Scope().IsAssetInScope(ip, 0) - if conf > 0 { - inscope = true - } + var inscope bool + _, conf := e.Session.Scope().IsAssetInScope(ip, 0) + if conf > 0 { + inscope = true + } - for _, rel := range rels { - if inscope { - if dom, err := publicsuffix.EffectiveTLDPlusOne(rel.ToAsset.Asset.Key()); err == nil && dom != "" { - if e.Session.Scope().AddDomain(dom) { - h.plugin.log.Info(fmt.Sprintf("[%s: %s] was added to the session scope", "FQDN", dom)) + for _, rel := range rels { + if inscope { + if dom, err := publicsuffix.EffectiveTLDPlusOne(rel.ToAsset.Asset.Key()); err == nil && dom != "" { + if e.Session.Scope().AddDomain(dom) { + h.plugin.log.Info(fmt.Sprintf("[%s: %s] was added to the session scope", "FQDN", dom)) + } + h.plugin.submitFQDN(e, dom, src) } - h.plugin.submitFQDN(e, dom, src) - } - } else if _, conf := e.Session.Scope().IsAssetInScope(rel.ToAsset.Asset, 0); conf > 0 { - if e.Session.Scope().Add(ip) { - size := 100 - if e.Session.Config().Active { - size = 250 + } else if _, conf := e.Session.Scope().IsAssetInScope(rel.ToAsset.Asset, 0); conf > 0 { + if e.Session.Scope().Add(ip) { + size := 100 + if e.Session.Config().Active { + size = 250 + } + h.plugin.submitIPAddresses(e, ip, src) + support.IPAddressSweep(e, ip, src, size, h.plugin.submitIPAddresses) + h.plugin.log.Info(fmt.Sprintf("[%s: %s] was added to the session scope", ip.AssetType(), ip.Key())) } - h.plugin.submitIPAddresses(e, ip, src) - support.IPAddressSweep(e, ip, src, size, h.plugin.submitIPAddresses) - h.plugin.log.Info(fmt.Sprintf("[%s: %s] was added to the session scope", ip.AssetType(), ip.Key())) } } } diff --git a/engine/plugins/horizontals/plugin.go b/engine/plugins/horizontals/plugin.go index 81e3ae6d0..96d8764c1 100644 --- a/engine/plugins/horizontals/plugin.go +++ b/engine/plugins/horizontals/plugin.go @@ -14,6 +14,7 @@ import ( "github.com/miekg/dns" "github.com/owasp-amass/amass/v4/engine/plugins/support" + "github.com/owasp-amass/amass/v4/engine/sessions/scope" et "github.com/owasp-amass/amass/v4/engine/types" amassnet "github.com/owasp-amass/amass/v4/utils/net" dbt "github.com/owasp-amass/asset-db/types" @@ -89,6 +90,56 @@ func (h *horizPlugin) Stop() { h.log.Info("Plugin stopped") } +func (h *horizPlugin) addAssociatedRelationship(e *et.Event, assocs []*scope.Association) { + for _, assoc := range assocs { + for _, impacted := range assoc.ImpactedAssets { + if match, conf := e.Session.Scope().IsAssetInScope(impacted.Asset, 0); conf > 0 && match != nil { + if a, hit := e.Session.Cache().GetAsset(match); hit && a != nil { + for _, assoc2 := range e.Session.Scope().AssetsWithAssociation(e.Session.Cache(), a) { + h.makeAssocRelationshipEntries(e, assoc.Match, assoc2) + } + } + } + } + } +} + +func (h *horizPlugin) makeAssocRelationshipEntries(e *et.Event, assoc, assoc2 *dbt.Asset) { + if assoc.ID == assoc2.ID { + return + } + + now := time.Now() + e.Session.Cache().SetRelation(&dbt.Relation{ + Type: "associated_with", + CreatedAt: now, + LastSeen: now, + FromAsset: assoc, + ToAsset: assoc2, + }) + e.Session.Cache().SetRelation(&dbt.Relation{ + Type: "associated_with", + CreatedAt: now, + LastSeen: now, + FromAsset: assoc2, + ToAsset: assoc, + }) + + done := make(chan struct{}, 1) + defer close(done) + + support.AppendToDBQueue(func() { + defer func() { done <- struct{}{} }() + + if e.Session.Done() { + return + } + + _, _ = e.Session.DB().Link(assoc, "associated_with", assoc2) + _, _ = e.Session.DB().Link(assoc2, "associated_with", assoc) + }) +} + func (h *horizPlugin) process(e *et.Event, assets []*dbt.Asset, src *dbt.Asset) { for _, asset := range assets { // check for new networks added to the scope diff --git a/engine/sessions/scope/assoc.go b/engine/sessions/scope/assoc.go index 6e38947fa..c2c179d72 100644 --- a/engine/sessions/scope/assoc.go +++ b/engine/sessions/scope/assoc.go @@ -45,7 +45,7 @@ func (s *Scope) IsAssociated(c cache.Cache, req *Association) ([]*Association, e } // related assets that provide association matching value - assocs := s.assetsWithAssociation(c, req.Submission) + assocs := s.AssetsWithAssociation(c, req.Submission) // are any of these assets in the current session scope? results := s.checkRelatedAssetsforAssoc(c, req, assocs) @@ -202,7 +202,7 @@ func (s *Scope) assetsRelatedToAssetWithAssoc(c cache.Cache, assoc *dbt.Asset) [ return results } -func (s *Scope) assetsWithAssociation(c cache.Cache, asset *dbt.Asset) []*dbt.Asset { +func (s *Scope) AssetsWithAssociation(c cache.Cache, asset *dbt.Asset) []*dbt.Asset { set := stringset.New(asset.ID) defer set.Close()