diff --git a/deepfence_agent/plugins/YaraHunter b/deepfence_agent/plugins/YaraHunter index c729ccb17f..395346ecce 160000 --- a/deepfence_agent/plugins/YaraHunter +++ b/deepfence_agent/plugins/YaraHunter @@ -1 +1 @@ -Subproject commit c729ccb17f3611052c271084e1bdfa84b55edcc5 +Subproject commit 395346ecce9946ce1a276f45b9fa7ad04ce9bfe2 diff --git a/deepfence_agent/plugins/cloud-scanner b/deepfence_agent/plugins/cloud-scanner index 9e300c1678..c25aedcfdb 160000 --- a/deepfence_agent/plugins/cloud-scanner +++ b/deepfence_agent/plugins/cloud-scanner @@ -1 +1 @@ -Subproject commit 9e300c167816acfd1326ae15614b320c92374c9e +Subproject commit c25aedcfdbc09a9deae9f40cdea9367a6b6ca658 diff --git a/deepfence_agent/plugins/yara-rules b/deepfence_agent/plugins/yara-rules index e5f8c8b78e..8217b51893 160000 --- a/deepfence_agent/plugins/yara-rules +++ b/deepfence_agent/plugins/yara-rules @@ -1 +1 @@ -Subproject commit e5f8c8b78e82bcc2f56a95ed77ed19d8dc8f6a66 +Subproject commit 8217b518934b556ee7f56e6e5fc3e05be8c8d9fa diff --git a/deepfence_server/handler/topology.go b/deepfence_server/handler/topology.go index 18c8259d65..87d06a6653 100644 --- a/deepfence_server/handler/topology.go +++ b/deepfence_server/handler/topology.go @@ -278,7 +278,7 @@ func graphToSummaries( return res } - for cp, crs := range graph.CloudServices { + for cp, crs := range graph.InternalCloudServices { for _, crStub := range crs { cr := string(crStub.ID) nodes[cr] = detailed.NodeSummary{ @@ -293,6 +293,14 @@ func graphToSummaries( nodes["in-the-internet"] = inboundInternetNode nodes["out-the-internet"] = outboundInternetNode + for _, n := range graph.ExternalCloudServices { + nodes[string(n.ID)] = detailed.NodeSummary{ + ID: string(n.ID), + Label: n.Name, + ImmediateParentID: "", + Type: "pseudo", + } + } for h, n := range graph.Processes { for _, idStub := range n { diff --git a/deepfence_server/ingesters/agent.go b/deepfence_server/ingesters/agent.go index edb6ec0831..2286bf2175 100644 --- a/deepfence_server/ingesters/agent.go +++ b/deepfence_server/ingesters/agent.go @@ -1434,7 +1434,7 @@ func resolveCloudService(connections []Connection, token string) []Connection { } infos, err := requestCloudInfo(context.Background(), ips, token) if err != nil || len(connections) != len(infos) { - log.Error().Err(err).Msgf("issue fetching cloud infos %d/%d", len(infos), len(connections)) + log.Error().Err(err).Msgf("issue fetching cloud infos %v/%v", infos, connections) return connections } for i := range infos { diff --git a/deepfence_server/reporters/graph/topology_reporter.go b/deepfence_server/reporters/graph/topology_reporter.go index fef9a53d5f..d7aa34fcb3 100644 --- a/deepfence_server/reporters/graph/topology_reporter.go +++ b/deepfence_server/reporters/graph/topology_reporter.go @@ -436,7 +436,7 @@ func extractResourceNodeIds(ids []interface{}) []NodeID { return res } -func (ntp *neo4jTopologyReporter) GetCloudServices( +func (ntp *neo4jTopologyReporter) GetInternalCloudServices( ctx context.Context, tx neo4j.ExplicitTransaction, cloudProvider []string, @@ -491,6 +491,46 @@ func (ntp *neo4jTopologyReporter) GetCloudServices( } +func (ntp *neo4jTopologyReporter) GetExternalCloudServices( + ctx context.Context, + tx neo4j.ExplicitTransaction, + cloudProvider []string, + cloudRegions []string, + fieldfilters mo.Option[reporters.FieldsFilters]) ([]NodeStub, error) { + + ctx, span := telemetry.NewSpan(ctx, "toploogy", "get-cloud-services") + defer span.End() + + res := []NodeStub{} + + r, err := tx.Run(ctx, ` + MATCH (n:Node) -[:CONNECTS]- (:Node) + WHERE n.cloud_provider = "internet" + RETURN n.node_id`, + map[string]interface{}{}, + ) + + if err != nil { + return res, err + } + records, err := r.Collect(ctx) + + if err != nil { + return res, err + } + + for _, record := range records { + nodeID := record.Values[0].(string) + res = append(res, + NodeStub{ + ID: NodeID(nodeID), + Name: nodeID, + }) + } + return res, nil + +} + func (ntp *neo4jTopologyReporter) GetPublicCloudResources( ctx context.Context, tx neo4j.ExplicitTransaction, @@ -921,8 +961,9 @@ type RenderedGraph struct { Connections []ConnectionSummary `json:"connections" required:"true"` // PublicCloudResources map[NodeID][]ResourceStub `json:"public-cloud-resources" required:"true"` // NonPublicCloudResources map[NodeID][]ResourceStub `json:"non-public-cloud-resources" required:"true"` - CloudServices map[NodeID][]ResourceStub `json:"cloud-services" required:"true"` - SkippedConnections bool `json:"skipped_connections" required:"true"` + InternalCloudServices map[NodeID][]ResourceStub `json:"cloud-services" required:"true"` + ExternalCloudServices []NodeStub `json:"external-cloud-services" required:"true"` + SkippedConnections bool `json:"skipped_connections" required:"true"` } type TopologyFilters struct { @@ -1239,7 +1280,11 @@ func (ntp *neo4jTopologyReporter) getGraph(ctx context.Context, filters Topology if err != nil { return res, err } - res.CloudServices, err = ntp.GetCloudServices(ctx, tx, cloudFilter, regionFilter, mo.None[reporters.FieldsFilters]()) + res.InternalCloudServices, err = ntp.GetInternalCloudServices(ctx, tx, cloudFilter, regionFilter, mo.None[reporters.FieldsFilters]()) + if err != nil { + return res, err + } + res.ExternalCloudServices, err = ntp.GetExternalCloudServices(ctx, tx, cloudFilter, regionFilter, mo.None[reporters.FieldsFilters]()) if err != nil { return res, err } diff --git a/deepfence_worker/cronjobs/neo4j.go b/deepfence_worker/cronjobs/neo4j.go index 8b10da8cf2..7d90488fdd 100644 --- a/deepfence_worker/cronjobs/neo4j.go +++ b/deepfence_worker/cronjobs/neo4j.go @@ -780,6 +780,7 @@ func LinkNodes(ctx context.Context, task *asynq.Task) error { AND NOT n.cloud_provider IS NULL AND NOT n.cloud_region IS NULL AND NOT n.node_id IN ["in-the-internet", "out-the-internet", "`+ConsoleAgentId+`"] + AND NOT n.cloud_provider = 'internet' WITH n LIMIT 50000 MERGE (cp:CloudProvider{node_id: n.cloud_provider}) MERGE (cr:CloudRegion{node_id: n.cloud_region}) @@ -816,7 +817,7 @@ func LinkNodes(ctx context.Context, task *asynq.Task) error { WHERE NOT exists((n) -[:ALIAS]-> ()) MERGE (t:ImageTag{node_id: n.docker_image_name + "_" + n.docker_image_tag}) MERGE (n) -[a:ALIAS]-> (t) - SET t.updated_at = TIMESTAMP(), + SET t.updated_at = TIMESTAMP(), a.updated_at = TIMESTAMP()`, map[string]interface{}{}, txConfig); err != nil { return err diff --git a/golang_deepfence_sdk b/golang_deepfence_sdk index d55cb83c33..226fd1e1e6 160000 --- a/golang_deepfence_sdk +++ b/golang_deepfence_sdk @@ -1 +1 @@ -Subproject commit d55cb83c3387857543eb9a0d656bc09b6cc84e59 +Subproject commit 226fd1e1e6bb7b9f4e179d6f738bb7cd9c55dd72