diff --git a/internal/db/arango.go b/internal/db/arango.go index 58d2ad7..f95b37f 100644 --- a/internal/db/arango.go +++ b/internal/db/arango.go @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2022 Dell Inc. + * Copyright 2024 Dell Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -20,6 +20,7 @@ import ( "github.com/arangodb/go-driver" "github.com/arangodb/go-driver/http" + "github.com/project-alvarium/alvarium-sdk-go/pkg/contracts" "github.com/project-alvarium/alvarium-sdk-go/pkg/interfaces" "github.com/project-alvarium/scoring-apps-go/internal/config" "github.com/project-alvarium/scoring-apps-go/pkg/documents" @@ -34,7 +35,10 @@ type ArangoClient struct { logger interfaces.Logger } -func NewArangoClient(configs []config.DatabaseInfo, logger interfaces.Logger) (*ArangoClient, error) { +func NewArangoClient( + configs []config.DatabaseInfo, + logger interfaces.Logger, +) (*ArangoClient, error) { client := ArangoClient{ logger: logger, } @@ -88,7 +92,7 @@ func (c *ArangoClient) QueryScore(ctx context.Context, key string) (documents.Sc } defer cursor.Close() - //There should only be one document returned here + // There should only be one document returned here var score documents.Score for { _, err := cursor.ReadDocument(ctx, &score) @@ -101,14 +105,36 @@ func (c *ArangoClient) QueryScore(ctx context.Context, key string) (documents.Sc return score, nil } -func (c *ArangoClient) QueryAnnotations(ctx context.Context, key string) ([]documents.Annotation, error) { +func (c *ArangoClient) QueryAnnotations( + ctx context.Context, + key string, +) ([]documents.Annotation, error) { db, err := c.instance.Database(ctx, c.cfg.DatabaseName) if err != nil { return nil, err } - query := "FOR a in annotations FILTER a.dataRef == @key RETURN a" + + // This query gets the data score (app layer), then checks all connected nodes + // with the "stack" edge, it should include all influencing CICD and OS scores. + // For each connected score node, the "tags" array is iterated on and all annotations + // that have a tag included in that array are returned by the query. This will work + // with all layer annotations. + query := ` + FOR score IN scores FILTER score.dataRef == @key + FOR v, e, p IN 1..1 ANY score._id GRAPH @graph + FILTER CONTAINS(e._id, @stack) + LET tags = v.tag + LET layer = v.layer + FOR tag IN tags + FOR annotation IN annotations + FILTER annotation.tag IN tags AND + (annotation.layer != @app OR annotation.dataRef == @key) + RETURN annotation + ` bindVars := map[string]interface{}{ - "key": key, + "key": key, + "stack": documents.EdgeStack, + "graph": c.cfg.GraphName, } cursor, err := db.Query(ctx, query, bindVars) if err != nil { @@ -118,14 +144,107 @@ func (c *ArangoClient) QueryAnnotations(ctx context.Context, key string) ([]docu var annotations []documents.Annotation for { - var doc documents.Annotation - _, err := cursor.ReadDocument(ctx, &doc) + var a documents.Annotation + _, err := cursor.ReadDocument(ctx, &a) if driver.IsNoMoreDocuments(err) { break - } else if err != nil { + } + if err != nil { return nil, err } - annotations = append(annotations, doc) + annotations = append(annotations, a) } + return annotations, nil } + +func (c *ArangoClient) QueryScoreByLayer( + ctx context.Context, + key string, + layer contracts.LayerType, +) ([]documents.Score, error) { + db, err := c.instance.Database(ctx, c.cfg.DatabaseName) + if err != nil { + return nil, err + } + + var query string + switch layer { + case contracts.Application: + query = `FOR s IN scores FILTER s.dataRef == @key AND s.layer == @layer RETURN [s]` + case contracts.CiCd: + query = `FOR appScore IN scores FILTER appScore.dataRef == @key + LET cicdScore = ( + FOR s IN scores FILTER + s.layer == @layer AND s.tag ANY IN appScore.tag + RETURN s + ) + RETURN cicdScore ` + case contracts.Os, contracts.Host: + query = `FOR a in annotations FILTER a.dataRef == @key LIMIT 1 + LET scores = (FOR s IN scores FILTER s.layer == @layer AND + a.host IN s.tag RETURN s) + RETURN scores` + + } + bindVars := map[string]interface{}{ + "key": key, + "layer": layer, + } + cursor, err := db.Query(ctx, query, bindVars) + if err != nil { + return nil, err + } + defer cursor.Close() + + var scores []documents.Score + for { + _, err := cursor.ReadDocument(ctx, &scores) + if driver.IsNoMoreDocuments(err) { + break + } else if err != nil { + return nil, err + } + } + + return scores, nil +} + +func (c *ArangoClient) FetchHosts(ctx context.Context) ([]string, error) { + db, err := c.instance.Database(ctx, c.cfg.DatabaseName) + if err != nil { + return nil, err + } + + query := `FOR a IN annotations FILTER a.layer == @app LET hosts = (a.host) RETURN DISTINCT hosts` + bindVars := map[string]interface{}{ + "app": string(contracts.Application), + } + cursor, err := db.Query(ctx, query, bindVars) + if err != nil { + return nil, err + } + defer cursor.Close() + + var hosts []string + for { + // returning 1 result will expect a string value, + // if multiple values, expects a []string value + if cursor.Count() > 1 { + _, err = cursor.ReadDocument(ctx, &hosts) + } else { + var host string + _, err = cursor.ReadDocument(ctx, &host) + if err == nil { + hosts = append(hosts, host) + } + } + if driver.IsNoMoreDocuments(err) { + break + } else if err != nil { + return nil, err + } + } + + return hosts, nil +} diff --git a/internal/hashprovider/types.go b/internal/hashprovider/types.go index 2727e42..2a69f01 100644 --- a/internal/hashprovider/types.go +++ b/internal/hashprovider/types.go @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2022 Dell Inc. + * Copyright 2024 Dell Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -17,11 +17,12 @@ package hashprovider import ( crypto "crypto/sha256" "encoding/hex" + "strings" ) func DeriveHash(data []byte) string { h := crypto.Sum256(data) hashEncoded := make([]byte, hex.EncodedLen(len(h))) hex.Encode(hashEncoded, h[:]) - return string(hashEncoded) + return strings.ToUpper(string(hashEncoded)) } diff --git a/internal/populator-api/routes.go b/internal/populator-api/routes.go index a32f348..c451e24 100644 --- a/internal/populator-api/routes.go +++ b/internal/populator-api/routes.go @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright 2022 Dell Inc. + * Copyright 2024 Dell Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -16,13 +16,17 @@ package populator_api import ( "encoding/json" + "errors" "fmt" "log/slog" "net/http" + "slices" "strconv" + "strings" "time" "github.com/gorilla/mux" + "github.com/project-alvarium/alvarium-sdk-go/pkg/contracts" "github.com/project-alvarium/alvarium-sdk-go/pkg/interfaces" "github.com/project-alvarium/scoring-apps-go/internal/db" "github.com/project-alvarium/scoring-apps-go/internal/hashprovider" @@ -41,22 +45,32 @@ func LoadRestRoutes(r *mux.Router, dbArango *db.ArangoClient, dbMongo *db.MongoP r.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { getIndexHandler(w, r, logger) - }).Methods(http.MethodGet) + }).Methods(http.MethodGet, http.MethodOptions) r.HandleFunc("/data/{limit:[0-9]+}", func(w http.ResponseWriter, r *http.Request) { - getSampleDataHandler(w, r, dbMongo, logger) - }).Methods(http.MethodGet) + getSampleDataHandler(w, r, dbMongo, dbArango, logger) + }).Methods(http.MethodGet, http.MethodOptions) r.HandleFunc("/data/count", func(w http.ResponseWriter, r *http.Request) { getDocumentCountHandler(w, r, dbMongo, logger) - }).Methods(http.MethodGet) + }).Methods(http.MethodGet, http.MethodOptions) r.HandleFunc("/data/{id}/annotations", func(w http.ResponseWriter, r *http.Request) { getAnnotationsHandler(w, r, dbMongo, dbArango, logger) - }).Methods(http.MethodGet) + }).Methods(http.MethodGet, http.MethodOptions) + + r.HandleFunc("/data/{id}/confidence", + func(w http.ResponseWriter, r *http.Request) { + getDataConfidence(w, r, dbMongo, dbArango, logger) + }).Methods(http.MethodGet, http.MethodOptions) + + r.HandleFunc("/hosts", + func(w http.ResponseWriter, r *http.Request) { + getHosts(w, r, dbArango, logger) + }).Methods(http.MethodGet, http.MethodOptions) } func getIndexHandler(w http.ResponseWriter, r *http.Request, logger interfaces.Logger) { @@ -89,7 +103,13 @@ func getDocumentCountHandler(w http.ResponseWriter, r *http.Request, dbMongo *db w.Write(b) } -func getSampleDataHandler(w http.ResponseWriter, r *http.Request, dbMongo *db.MongoProvider, logger interfaces.Logger) { +func getSampleDataHandler( + w http.ResponseWriter, + r *http.Request, + dbMongo *db.MongoProvider, + dbArango *db.ArangoClient, + logger interfaces.Logger, +) { defer r.Body.Close() vars := mux.Vars(r) @@ -100,6 +120,7 @@ func getSampleDataHandler(w http.ResponseWriter, r *http.Request, dbMongo *db.Mo w.Write([]byte(err.Error())) return } + results, err := dbMongo.QueryMostRecent(r.Context(), limit) if err != nil { logger.Error(err.Error()) @@ -107,9 +128,49 @@ func getSampleDataHandler(w http.ResponseWriter, r *http.Request, dbMongo *db.Mo w.Write([]byte(err.Error())) return } + + // Applying a host filter on the data if supplied + + host := r.URL.Query().Get("host") var viewModels []responses.DataViewModel - for _, i := range results { - viewModels = append(viewModels, models.ViewModelFromMongoRecord(i)) + for _, record := range results { + // skip host filter if not supplied + if host == "" { + viewModels = append(viewModels, models.ViewModelFromMongoRecord(record)) + continue + } + // Current approach is getting the dataRef by hashing + // the mongo record, then fetching annotations by that + // dataRef and finding their host + sampleData := models.SampleFromMongoRecord(record) + b, _ := json.Marshal(sampleData) + key := hashprovider.DeriveHash(b) + + annotations, err := dbArango.QueryAnnotations(r.Context(), key) + if err != nil { + logger.Error("failed to filter data by hosts : " + err.Error()) + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + if len(annotations) == 0 { + err := errors.New("failed to filter data by hosts : annotations required to find hosts") + logger.Error(err.Error()) + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + + for _, annotation := range annotations { + if strings.EqualFold(annotation.Host, host) { + exists := slices.ContainsFunc(viewModels, func(model responses.DataViewModel) bool { + return strings.EqualFold(model.Id.String(), record.Id) + }) + if !exists { + viewModels = append(viewModels, models.ViewModelFromMongoRecord(record)) + } + } + } } response := responses.DataListResponse{ @@ -167,3 +228,86 @@ func getAnnotationsHandler(w http.ResponseWriter, r *http.Request, dbMongo *db.M w.WriteHeader(http.StatusOK) w.Write(b) } + +func getDataConfidence( + w http.ResponseWriter, + r *http.Request, + dbMongo *db.MongoProvider, + dbArango *db.ArangoClient, + logger interfaces.Logger, +) { + defer r.Body.Close() + + vars := mux.Vars(r) + id := vars["id"] + + layerRaw := r.URL.Query().Get("layer") + var layer contracts.LayerType + if layerRaw == "" { + layer = contracts.Application + } else { + layer = contracts.LayerType(layerRaw) + } + + if !layer.Validate() { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("Bad layer value: " + layerRaw)) + return + } + + record, err := dbMongo.FetchById(r.Context(), id) + if err != nil { + logger.Error(err.Error()) + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + + data := models.SampleFromMongoRecord(record) + b, _ := json.Marshal(data) + key := hashprovider.DeriveHash(b) + + scores, err := dbArango.QueryScoreByLayer(r.Context(), key, layer) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + s, err := json.Marshal(scores) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + w.Header().Add(headerKeyContentType, headerValueJson) + w.Header().Add(headerCORS, headerCORSValue) + w.WriteHeader(http.StatusOK) + w.Write(s) +} + +func getHosts( + w http.ResponseWriter, + r *http.Request, + dbArango *db.ArangoClient, + logger interfaces.Logger, +) { + hosts, err := dbArango.FetchHosts(r.Context()) + if err != nil { + logger.Error(err.Error()) + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + + payload, err := json.Marshal(hosts) + if err != nil { + logger.Error(err.Error()) + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + w.Header().Add(headerKeyContentType, headerValueJson) + w.Header().Add(headerCORS, headerCORSValue) + w.WriteHeader(http.StatusOK) + w.Write(payload) +} diff --git a/pkg/documents/types.go b/pkg/documents/types.go index cb38ea3..4ad2196 100644 --- a/pkg/documents/types.go +++ b/pkg/documents/types.go @@ -71,13 +71,13 @@ func NewAnnotation(a contracts.Annotation) Annotation { // Score represents a document in the "score" vertex collection type Score struct { - Key ulid.ULID `json:"_key,omitempty"` // Key uniquely identifies the document in the database - DataRef string `json:"dataRef,omitempty"` // DataRef points to the key of the data being annotated - Passed int `json:"score,omitempty"` // Passed indicates how many of the annotations for a given dataRef were Satisfied - Count int `json:"count,omitempty"` // Count indicates the total number of annotations applicable to a dataRef - Policy string `json:"policy,omitempty"` // Policy will indicate some version of the policy used to calculate confidence - Confidence float64 `json:"confidence,omitempty"` // Confidence is the percentage of trust in the dataRef - Timestamp time.Time `json:"timestamp,omitempty"` // Timestamp indicates when the score was calculated + Key ulid.ULID `json:"_key,omitempty"` // Key uniquely identifies the document in the database + DataRef string `json:"dataRef,omitempty"` // DataRef points to the key of the data being annotated + Passed int `json:"score"` // Passed indicates how many of the annotations for a given dataRef were Satisfied + Count int `json:"count"` // Count indicates the total number of annotations applicable to a dataRef + Policy string `json:"policy,omitempty"` // Policy will indicate some version of the policy used to calculate confidence + Confidence float64 `json:"confidence"` // Confidence is the percentage of trust in the dataRef + Timestamp time.Time `json:"timestamp,omitempty"` // Timestamp indicates when the score was calculated Tag []string `json:"tag,omitempty"` Layer contracts.LayerType `json:"layer,omitempty"` }