From 49c17283fd2dcba3bae7a75e77e03cb5f3a7777d Mon Sep 17 00:00:00 2001 From: Siddarth Kumar Date: Sat, 22 Apr 2023 18:56:07 +0530 Subject: [PATCH 1/7] add rds snapshot resource --- providers/aws/aws.go | 1 + providers/aws/rds/snapshots.go | 57 ++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 providers/aws/rds/snapshots.go diff --git a/providers/aws/aws.go b/providers/aws/aws.go index 24a42f2cd..567efceb5 100644 --- a/providers/aws/aws.go +++ b/providers/aws/aws.go @@ -63,6 +63,7 @@ func listOfSupportedServices() []providers.FetchDataFunction { kms.Keys, rds.Clusters, rds.Instances, + rds.Snapshots, elb.LoadBalancers, efs.ElasticFileStorage, apigateway.Apis, diff --git a/providers/aws/rds/snapshots.go b/providers/aws/rds/snapshots.go new file mode 100644 index 000000000..b22533e6f --- /dev/null +++ b/providers/aws/rds/snapshots.go @@ -0,0 +1,57 @@ +package rds + +import ( + "context" + "fmt" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/rds" + "github.com/tailwarden/komiser/models" + "github.com/tailwarden/komiser/providers" +) + +func Snapshots(ctx context.Context, client providers.ProviderClient) ([]models.Resource, error) { + var config rds.DescribeDBSnapshotsInput + resources := make([]models.Resource, 0) + rdsClient := rds.NewFromConfig(*client.AWSClient) + + for { + output, err := rdsClient.DescribeDBSnapshots(ctx, &config) + if err != nil { + return resources, err + } + + for _, snapshot := range output.DBSnapshots { + tags := make([]models.Tag, 0) + for _, tag := range snapshot.TagList { + tags = append(tags, models.Tag{ + Key: *tag.Key, + Value: *tag.Value, + }) + } + + _snapshotName := *snapshot.DBSnapshotIdentifier + + resources = append(resources, models.Resource{ + Provider: "AWS", + Account: client.Name, + Service: "RDS Snapshot", + Region: client.AWSClient.Region, + ResourceId: *snapshot.DBSnapshotArn, + Name: _snapshotName, + FetchedAt: time.Now(), + Tags: tags, + Link: fmt.Sprintf("https://%s.console.aws.amazon.com/rds/home?region=%s#snapshot:id=%s", client.AWSClient.Region, client.AWSClient.Region, *snapshot.DBSnapshotIdentifier), + }) + } + + if aws.ToString(output.Marker) == "" { + break + } + + config.Marker = output.Marker + } + + return resources, nil +} \ No newline at end of file From 19ba9479c022f11353abd92300fee9f0b19bb1be Mon Sep 17 00:00:00 2001 From: Kolawole Ojo Date: Sun, 30 Apr 2023 19:21:37 +0100 Subject: [PATCH 2/7] Stub function from VPC Endpoint --- providers/aws/ec2/vpc-peering-connection.go | 78 +++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 providers/aws/ec2/vpc-peering-connection.go diff --git a/providers/aws/ec2/vpc-peering-connection.go b/providers/aws/ec2/vpc-peering-connection.go new file mode 100644 index 000000000..ae1e53c9e --- /dev/null +++ b/providers/aws/ec2/vpc-peering-connection.go @@ -0,0 +1,78 @@ +package ec2 + +import ( + "context" + "fmt" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" + . "github.com/tailwarden/komiser/models" + . "github.com/tailwarden/komiser/providers" +) + +func VpcPeeringConnection(ctx context.Context, client ProviderClient) ([]Resource, error) { + var config ec2.DescribeVpcEndpointsInput + resources := make([]Resource, 0) + ec2Client := ec2.NewFromConfig(*client.AWSClient) + + for { + output, err := ec2Client.DescribeVpcEndpoints(ctx, &config) + if err != nil { + return resources, err + } + + for _, vpcEndpoint := range output.VpcEndpoints { + name := "" + tags := make([]Tag, 0) + for _, tag := range vpcEndpoint.Tags { + if *tag.Key == "Name" { + name = *tag.Value + } + tags = append(tags, Tag{ + Key: *tag.Key, + Value: *tag.Value, + }) + } + + resources = append(resources, Resource{ + Provider: "AWS", + Account: client.Name, + Service: "VPC Peering Connection", + Region: client.AWSClient.Region, + Name: name, + ResourceId: *vpcEndpoint.VpcEndpointId, + CreatedAt: *vpcEndpoint.CreationTimestamp, + FetchedAt: time.Now(), + Cost: 0, + Tags: tags, + Link: fmt.Sprintf( + "https:/%s.console.aws.amazon.com/vpc/home?region=%s#EndpointDetails:vpcEndpointId=%s", + client.AWSClient.Region, + client.AWSClient.Region, + *vpcEndpoint.VpcEndpointId, + ), + Metadata: map[string]string{ + "VpcEndpointType": string(vpcEndpoint.VpcEndpointType), + "ServiceName": string(*vpcEndpoint.ServiceName), + }, + }) + } + + if aws.ToString(output.NextToken) == "" { + break + } + + config.NextToken = output.NextToken + } + log.WithFields(log.Fields{ + "provider": "AWS", + "account": client.Name, + "region": client.AWSClient.Region, + "service": "VPC Peering Connection", + "resources": len(resources), + }).Info("Fetched resources") + return resources, nil +} From 82ca71f23bd9f44e11b569f15dfdb60854ce00d8 Mon Sep 17 00:00:00 2001 From: Kolawole Ojo Date: Mon, 1 May 2023 08:34:18 +0100 Subject: [PATCH 3/7] Added Support for VPC Peering Connections --- providers/aws/aws.go | 1 + providers/aws/ec2/vpc-peering-connection.go | 21 ++++++++------------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/providers/aws/aws.go b/providers/aws/aws.go index 46b850f0f..bc59a924a 100644 --- a/providers/aws/aws.go +++ b/providers/aws/aws.go @@ -80,6 +80,7 @@ func listOfSupportedServices() []providers.FetchDataFunction { ec2.PlacementGroups, systemsmanager.MaintenanceWindows, ec2.VpcEndpoints, + ec2.VpcPeeringConnections, } } diff --git a/providers/aws/ec2/vpc-peering-connection.go b/providers/aws/ec2/vpc-peering-connection.go index ae1e53c9e..878db98a5 100644 --- a/providers/aws/ec2/vpc-peering-connection.go +++ b/providers/aws/ec2/vpc-peering-connection.go @@ -13,21 +13,21 @@ import ( . "github.com/tailwarden/komiser/providers" ) -func VpcPeeringConnection(ctx context.Context, client ProviderClient) ([]Resource, error) { - var config ec2.DescribeVpcEndpointsInput +func VpcPeeringConnections(ctx context.Context, client ProviderClient) ([]Resource, error) { + var config ec2.DescribeVpcPeeringConnectionsInput resources := make([]Resource, 0) ec2Client := ec2.NewFromConfig(*client.AWSClient) for { - output, err := ec2Client.DescribeVpcEndpoints(ctx, &config) + output, err := ec2Client.DescribeVpcPeeringConnections(ctx, &config) if err != nil { return resources, err } - for _, vpcEndpoint := range output.VpcEndpoints { + for _, vpcPeeringConnection := range output.VpcPeeringConnections { name := "" tags := make([]Tag, 0) - for _, tag := range vpcEndpoint.Tags { + for _, tag := range vpcPeeringConnection.Tags { if *tag.Key == "Name" { name = *tag.Value } @@ -43,21 +43,16 @@ func VpcPeeringConnection(ctx context.Context, client ProviderClient) ([]Resourc Service: "VPC Peering Connection", Region: client.AWSClient.Region, Name: name, - ResourceId: *vpcEndpoint.VpcEndpointId, - CreatedAt: *vpcEndpoint.CreationTimestamp, + ResourceId: *vpcPeeringConnection.VpcPeeringConnectionId, FetchedAt: time.Now(), Cost: 0, Tags: tags, Link: fmt.Sprintf( - "https:/%s.console.aws.amazon.com/vpc/home?region=%s#EndpointDetails:vpcEndpointId=%s", + "https:/%s.console.aws.amazon.com/vpc/home?region=%s#PeeringConnectionDetails:VpcPeeringConnectionId=%s", client.AWSClient.Region, client.AWSClient.Region, - *vpcEndpoint.VpcEndpointId, + *vpcPeeringConnection.VpcPeeringConnectionId, ), - Metadata: map[string]string{ - "VpcEndpointType": string(vpcEndpoint.VpcEndpointType), - "ServiceName": string(*vpcEndpoint.ServiceName), - }, }) } From 14f1201efe9f18484c9de99bbd5456a8c070154e Mon Sep 17 00:00:00 2001 From: Siddarth Kumar Date: Tue, 2 May 2023 15:36:20 +0530 Subject: [PATCH 4/7] chore: update hourly price calculation - ec2 & rds --- models/price.go | 12 ++++++++---- providers/aws/ec2/instances.go | 21 +++++++++++++++------ providers/aws/rds/instances.go | 21 +++++++++++++++------ 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/models/price.go b/models/price.go index f07df5ac4..de3d2841d 100644 --- a/models/price.go +++ b/models/price.go @@ -4,9 +4,13 @@ type PricingResult struct { Product struct { Sku string `json:"sku"` } `json:"product"` - Terms map[string]map[string]map[string]map[string]struct { - PricePerUnit struct { - USD string `json:"USD"` - } `json:"pricePerUnit"` + Terms struct { + OnDemand map[string]struct { + PriceDimensions map[string]struct { + PricePerUnit struct { + USD string `json:"USD"` + } `json:"pricePerUnit"` + } `json:"priceDimensions"` + } `json:"OnDemand"` } `json:"terms"` } diff --git a/providers/aws/ec2/instances.go b/providers/aws/ec2/instances.go index 74af1fa80..1936c180d 100644 --- a/providers/aws/ec2/instances.go +++ b/providers/aws/ec2/instances.go @@ -101,17 +101,26 @@ func Instances(ctx context.Context, client providers.ProviderClient) ([]models.R hourlyCost := 0.0 if pricingOutput != nil && len(pricingOutput.PriceList) > 0 { - b, _ := json.Marshal(pricingOutput.PriceList[0]) - s, _ := strconv.Unquote(string(b)) pricingResult := models.PricingResult{} - err = json.Unmarshal([]byte(s), &pricingResult) + err := json.Unmarshal([]byte(pricingOutput.PriceList[0]), &pricingResult) if err != nil { - log.WithError(err).Error("could not unmarshal") + log.Fatalf("Failed to unmarshal JSON: %v", err) } - hourlyCostRaw := pricingResult.Terms["OnDemand"][fmt.Sprintf("%s.JRTCKXETXF", pricingResult.Product.Sku)]["priceDimensions"][fmt.Sprintf("%s.JRTCKXETXF.6YS6EN2CT7", pricingResult.Product.Sku)].PricePerUnit.USD - hourlyCost, _ = strconv.ParseFloat(hourlyCostRaw, 64) + for _, onDemand := range pricingResult.Terms.OnDemand { + for _, priceDimension := range onDemand.PriceDimensions { + hourlyCost, err = strconv.ParseFloat(priceDimension.PricePerUnit.USD, 64) + if err != nil { + log.Fatalf("Failed to parse hourly cost: %v", err) + } + break + } + break + } + + //log.Printf("Hourly cost EC2: %f", hourlyCost) + } monthlyCost := float64(hourlyUsage) * hourlyCost diff --git a/providers/aws/rds/instances.go b/providers/aws/rds/instances.go index 6ddf407b8..f2b74bd46 100644 --- a/providers/aws/rds/instances.go +++ b/providers/aws/rds/instances.go @@ -85,17 +85,26 @@ func Instances(ctx context.Context, client providers.ProviderClient) ([]models.R hourlyCost := 0.0 if pricingOutput != nil && len(pricingOutput.PriceList) > 0 { - b, _ := json.Marshal(pricingOutput.PriceList[0]) - s, _ := strconv.Unquote(string(b)) + log.Infof(`Raw pricingOutput.PriceList[0] : %s`, pricingOutput.PriceList[0]) pricingResult := models.PricingResult{} - err = json.Unmarshal([]byte(s), &pricingResult) + err := json.Unmarshal([]byte(pricingOutput.PriceList[0]), &pricingResult) if err != nil { - log.WithError(err).Error("could not unmarshal") + log.Fatalf("Failed to unmarshal JSON: %v", err) } - hourlyCostRaw := pricingResult.Terms["OnDemand"][fmt.Sprintf("%s.JRTCKXETXF", pricingResult.Product.Sku)]["priceDimensions"][fmt.Sprintf("%s.JRTCKXETXF.6YS6EN2CT7", pricingResult.Product.Sku)].PricePerUnit.USD - hourlyCost, _ = strconv.ParseFloat(hourlyCostRaw, 64) + for _, onDemand := range pricingResult.Terms.OnDemand { + for _, priceDimension := range onDemand.PriceDimensions { + hourlyCost, err = strconv.ParseFloat(priceDimension.PricePerUnit.USD, 64) + if err != nil { + log.Fatalf("Failed to parse hourly cost: %v", err) + } + break + } + break + } + + //log.Printf("Hourly cost RDS: %f", hourlyCost) } monthlyCost := float64(hourlyUsage) * hourlyCost From dc863859f844f595c034d44b97012b98ee7c0450 Mon Sep 17 00:00:00 2001 From: Mohamed Labouardy Date: Thu, 4 May 2023 11:10:33 +0200 Subject: [PATCH 5/7] feat: track tagging & fix naming of files --- handlers/tags_handler.go | 8 ++++++++ .../aws/ec2/{vpc-endpoints.go => vpc_endpoints.go} | 0 ...peering-connection.go => vpc_peering_connection.go} | 0 providers/aws/rds/snapshots.go | 10 +++++++++- 4 files changed, 17 insertions(+), 1 deletion(-) rename providers/aws/ec2/{vpc-endpoints.go => vpc_endpoints.go} (100%) rename providers/aws/ec2/{vpc-peering-connection.go => vpc_peering_connection.go} (100%) diff --git a/handlers/tags_handler.go b/handlers/tags_handler.go index d3ea05518..d199edeac 100644 --- a/handlers/tags_handler.go +++ b/handlers/tags_handler.go @@ -28,6 +28,10 @@ func (handler *ApiHandler) BulkUpdateTagsHandler(c *gin.Context) { } } + if handler.telemetry { + handler.analytics.TrackEvent("tagging_resources", nil) + } + c.JSON(http.StatusCreated, gin.H{"message": "tags has been successfuly updated"}) } @@ -56,5 +60,9 @@ func (handler *ApiHandler) UpdateTagsHandler(c *gin.Context) { return } + if handler.telemetry { + handler.analytics.TrackEvent("tagging_resources", nil) + } + c.JSON(http.StatusCreated, gin.H{"message": "tags has been successfuly updated"}) } diff --git a/providers/aws/ec2/vpc-endpoints.go b/providers/aws/ec2/vpc_endpoints.go similarity index 100% rename from providers/aws/ec2/vpc-endpoints.go rename to providers/aws/ec2/vpc_endpoints.go diff --git a/providers/aws/ec2/vpc-peering-connection.go b/providers/aws/ec2/vpc_peering_connection.go similarity index 100% rename from providers/aws/ec2/vpc-peering-connection.go rename to providers/aws/ec2/vpc_peering_connection.go diff --git a/providers/aws/rds/snapshots.go b/providers/aws/rds/snapshots.go index b22533e6f..84bfdcfb7 100644 --- a/providers/aws/rds/snapshots.go +++ b/providers/aws/rds/snapshots.go @@ -7,6 +7,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/rds" + log "github.com/sirupsen/logrus" "github.com/tailwarden/komiser/models" "github.com/tailwarden/komiser/providers" ) @@ -53,5 +54,12 @@ func Snapshots(ctx context.Context, client providers.ProviderClient) ([]models.R config.Marker = output.Marker } + log.WithFields(log.Fields{ + "provider": "AWS", + "account": client.Name, + "region": client.AWSClient.Region, + "service": "RDS Snapshot", + "resources": len(resources), + }).Info("Fetched resources") return resources, nil -} \ No newline at end of file +} From 5b2f638c5fcaaedfa88a0639580ee9b8411940b8 Mon Sep 17 00:00:00 2001 From: Mohamed Labouardy Date: Thu, 4 May 2023 11:29:17 +0200 Subject: [PATCH 6/7] feat: track global stats if opt-in --- handlers/dashboard_handler.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/handlers/dashboard_handler.go b/handlers/dashboard_handler.go index f51ace2c5..aa8dc861d 100644 --- a/handlers/dashboard_handler.go +++ b/handlers/dashboard_handler.go @@ -63,6 +63,15 @@ func (handler *ApiHandler) DashboardStatsHandler(c *gin.Context) { Accounts: accounts.Count, } + if handler.telemetry { + handler.analytics.TrackEvent("global_stats", map[string]interface{}{ + "regions": regions.Count, + "resources": resources.Count, + "accounts": accounts.Count, + "cost": cost.Sum, + }) + } + c.JSON(http.StatusOK, output) } From 95c77ca1990a4234704c4a8489922b836a8128c3 Mon Sep 17 00:00:00 2001 From: Mohamed Labouardy Date: Fri, 5 May 2023 10:35:43 +0200 Subject: [PATCH 7/7] chore: bump v3.0.15 --- dashboard/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboard/package.json b/dashboard/package.json index 1f55c0796..a9753a84e 100644 --- a/dashboard/package.json +++ b/dashboard/package.json @@ -1,6 +1,6 @@ { "name": "komiser-dashboard", - "version": "3.0.14", + "version": "3.0.15", "private": true, "scripts": { "dev": "next dev -p 3002",