add external service to graph

deepfence_server/handler/topology.go

@@ -278,7 +278,7 @@ func graphToSummaries(
 		return res
 	}
 
-	for cp, crs := range graph.CloudServices {
+	for cp, crs := range graph.InternalCloudServices {
 		for _, crStub := range crs {
 			cr := string(crStub.ID)
 			nodes[cr] = detailed.NodeSummary{
@@ -293,6 +293,14 @@ func graphToSummaries(
 
 	nodes["in-the-internet"] = inboundInternetNode
 	nodes["out-the-internet"] = outboundInternetNode
+	for _, n := range graph.ExternalCloudServices {
+		nodes[string(n.ID)] = detailed.NodeSummary{
+			ID:                string(n.ID),
+			Label:             n.Name,
+			ImmediateParentID: "",
+			Type:              "pseudo",
+		}
+	}
 
 	for h, n := range graph.Processes {
 		for _, idStub := range n {

deepfence_server/ingesters/agent.go

@@ -1424,21 +1424,23 @@ func requestCloudInfo(ctx context.Context, strIps []string, token string) ([]Clo
 
 func resolveCloudService(connections []Connection, token string) []Connection {
 	ips := []string{}
+	ids := []int{} 
 	for i := range connections {
 		if connections[i].rightIP != nil {
 			ips = append(ips, *connections[i].rightIP)
+			ids = append(ids, i)
 		}
 	}
 	if len(ips) == 0 {
 		return connections
 	}
 	infos, err := requestCloudInfo(context.Background(), ips, token)
-	if err != nil || len(connections) != len(infos) {
-		log.Error().Err(err).Msgf("issue fetching cloud infos %d/%d", len(infos), len(connections))
+	if err != nil || len(ids) != len(infos) {
+		log.Error().Err(err).Msgf("issue fetching cloud infos %v/%v", infos, connections)
 		return connections
 	}
 	for i := range infos {
-		connections[i].destination = infos[i].NodeID()
+		connections[ids[i]].destination = infos[i].NodeID()
 	}
 	return connections
 }

deepfence_server/reporters/graph/topology_reporter.go

@@ -436,7 +436,7 @@ func extractResourceNodeIds(ids []interface{}) []NodeID {
 	return res
 }
 
-func (ntp *neo4jTopologyReporter) GetCloudServices(
+func (ntp *neo4jTopologyReporter) GetInternalCloudServices(
 	ctx context.Context,
 	tx neo4j.ExplicitTransaction,
 	cloudProvider []string,
@@ -491,6 +491,46 @@ func (ntp *neo4jTopologyReporter) GetCloudServices(
 
 }
 
+func (ntp *neo4jTopologyReporter) GetExternalCloudServices(
+	ctx context.Context,
+	tx neo4j.ExplicitTransaction,
+	cloudProvider []string,
+	cloudRegions []string,
+	fieldfilters mo.Option[reporters.FieldsFilters]) ([]NodeStub, error) {
+
+	ctx, span := telemetry.NewSpan(ctx, "toploogy", "get-cloud-services")
+	defer span.End()
+
+	res := []NodeStub{}
+
+	r, err := tx.Run(ctx, `
+		MATCH (n:Node) -[:CONNECTS]- (:Node)
+		WHERE n.cloud_provider = "internet"
+		RETURN n.node_id`,
+		map[string]interface{}{},
+	)
+
+	if err != nil {
+		return res, err
+	}
+	records, err := r.Collect(ctx)
+
+	if err != nil {
+		return res, err
+	}
+
+	for _, record := range records {
+		nodeID := record.Values[0].(string)
+		res = append(res,
+			NodeStub{
+				ID:   NodeID(nodeID),
+				Name: nodeID,
+			})
+	}
+	return res, nil
+
+}
+
 func (ntp *neo4jTopologyReporter) GetPublicCloudResources(
 	ctx context.Context,
 	tx neo4j.ExplicitTransaction,
@@ -921,8 +961,9 @@ type RenderedGraph struct {
 	Connections []ConnectionSummary   `json:"connections" required:"true"`
 	// PublicCloudResources    map[NodeID][]ResourceStub `json:"public-cloud-resources" required:"true"`
 	// NonPublicCloudResources map[NodeID][]ResourceStub `json:"non-public-cloud-resources" required:"true"`
-	CloudServices      map[NodeID][]ResourceStub `json:"cloud-services" required:"true"`
-	SkippedConnections bool                      `json:"skipped_connections" required:"true"`
+	InternalCloudServices map[NodeID][]ResourceStub `json:"cloud-services" required:"true"`
+	ExternalCloudServices []NodeStub                `json:"external-cloud-services" required:"true"`
+	SkippedConnections    bool                      `json:"skipped_connections" required:"true"`
 }
 
 type TopologyFilters struct {
@@ -1239,7 +1280,11 @@ func (ntp *neo4jTopologyReporter) getGraph(ctx context.Context, filters Topology
 	if err != nil {
 		return res, err
 	}
-	res.CloudServices, err = ntp.GetCloudServices(ctx, tx, cloudFilter, regionFilter, mo.None[reporters.FieldsFilters]())
+	res.InternalCloudServices, err = ntp.GetInternalCloudServices(ctx, tx, cloudFilter, regionFilter, mo.None[reporters.FieldsFilters]())
+	if err != nil {
+		return res, err
+	}
+	res.ExternalCloudServices, err = ntp.GetExternalCloudServices(ctx, tx, cloudFilter, regionFilter, mo.None[reporters.FieldsFilters]())
 	if err != nil {
 		return res, err
 	}

deepfence_worker/cronjobs/neo4j.go

@@ -780,6 +780,7 @@ func LinkNodes(ctx context.Context, task *asynq.Task) error {
 		AND NOT n.cloud_provider IS NULL
 		AND NOT n.cloud_region IS NULL
 		AND NOT n.node_id IN ["in-the-internet", "out-the-internet", "`+ConsoleAgentId+`"]
+		AND NOT n.cloud_provider = 'internet'
 		WITH n LIMIT 50000
 		MERGE (cp:CloudProvider{node_id: n.cloud_provider})
 		MERGE (cr:CloudRegion{node_id: n.cloud_region})
@@ -816,7 +817,7 @@ func LinkNodes(ctx context.Context, task *asynq.Task) error {
 		WHERE NOT exists((n) -[:ALIAS]-> ())
 		MERGE (t:ImageTag{node_id: n.docker_image_name + "_" + n.docker_image_tag})
 		MERGE (n) -[a:ALIAS]-> (t)
-		SET t.updated_at = TIMESTAMP(), 
+		SET t.updated_at = TIMESTAMP(),
 			a.updated_at = TIMESTAMP()`,
 		map[string]interface{}{}, txConfig); err != nil {
 		return err