From 72bdad489ad9285aa0227f82b1ae23762ccc1822 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Tue, 1 Oct 2024 13:07:38 +0200 Subject: [PATCH 1/3] provide-consumed-resources-from-prometheus (#676) * Added resources endpoint * Added prometheus client * Added multiple prometheus queries * Extracted prometheus metrics logic * Corrected min-max-avg values * Added comment and correct URL * Added query arguments * Added query arguments * Added extra query arguments * Added extra query arguments * Updated queries * Cleanup * Cleanup * Extracted prometheus handler * Adding unit-tests * Adding unit-tests * Adding unit-tests * Adding unit-tests * Adding unit-tests * Adding unit-tests * Updated refs * Renamed var * Send only raw values * Always remove zero values * Updated objects * Tuned queries * Corrected summary logic * Updated versions * Fixed tests * Fixed tests * Delete file * Moved the method --- .vscode/launch.json | 1 + Makefile | 2 + README.md | 5 + api/applications/applications_controller.go | 73 +++++- .../applications_controller_test.go | 227 +++++++++++++----- .../applications_handler_config.go | 2 + api/applications/models/used_resources.go | 54 +++++ api/metrics/internal/prometheus_defaults.go | 13 + api/metrics/mock/prometheus_client_mock.go | 53 ++++ api/metrics/mock/prometheus_handler_mock.go | 52 ++++ api/metrics/prometheus_client.go | 78 ++++++ api/metrics/prometheus_handler.go | 127 ++++++++++ api/metrics/prometheus_handler_test.go | 173 +++++++++++++ go.mod | 6 +- go.sum | 6 +- main.go | 19 +- radixconfig.yaml | 1 + swaggerui/html/swagger.json | 130 ++++++++++ 18 files changed, 950 insertions(+), 72 deletions(-) create mode 100644 api/applications/models/used_resources.go create mode 100644 api/metrics/internal/prometheus_defaults.go create mode 100644 api/metrics/mock/prometheus_client_mock.go create mode 100644 api/metrics/mock/prometheus_handler_mock.go create mode 100644 api/metrics/prometheus_client.go create mode 100644 api/metrics/prometheus_handler.go create mode 100644 api/metrics/prometheus_handler_test.go diff --git a/.vscode/launch.json b/.vscode/launch.json index 63fdd4e0..92558aa9 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -22,6 +22,7 @@ "REQUIRE_APP_CONFIGURATION_ITEM": "true", "REQUIRE_APP_AD_GROUPS": "true", "RADIX_ENVIRONMENT":"qa", + "PROMETHEUS_URL":"http://localhost:9091", "RADIX_APP":"radix-api", "LOG_LEVEL":"info", "LOG_PRETTY":"true" diff --git a/Makefile b/Makefile index 573f4abe..ae72015e 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,8 @@ build: $(BINS) mocks: bootstrap mockgen -source ./api/buildstatus/models/buildstatus.go -destination ./api/test/mock/buildstatus_mock.go -package mock mockgen -source ./api/deployments/deployment_handler.go -destination ./api/deployments/mock/deployment_handler_mock.go -package mock + mockgen -source ./api/metrics/prometheus_handler.go -destination ./api/metrics/mock/prometheus_handler_mock.go -package mock + mockgen -source ./api/metrics/prometheus_client.go -destination ./api/metrics/mock/prometheus_client_mock.go -package mock mockgen -source ./api/environments/job_handler.go -destination ./api/environments/mock/job_handler_mock.go -package mock mockgen -source ./api/environments/environment_handler.go -destination ./api/environments/mock/environment_handler_mock.go -package mock mockgen -source ./api/utils/tlsvalidation/interface.go -destination ./api/utils/tlsvalidation/mock/tls_secret_validator_mock.go -package mock diff --git a/README.md b/README.md index 210e279a..5e41a250 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,10 @@ The following env vars are needed. Useful default values in brackets. - `RADIX_CONTAINER_REGISTRY` - (`radixdev.azurecr.io`) - `PIPELINE_IMG_TAG` - (`master-latest`) - `TEKTON_IMG_TAG` - (`release-latest`) +- `PROMETHEUS_URL` - `http://localhost:9091` use this to get Prometheus metrics running the following command (the local port 9090 is used by the API server `/metrics` endpoint, in-cluster URL is http://prometheus-operator-prometheus.monitor.svc.cluster.local:9090): + ``` + kubectl -n monitor port-forward svc/prometheus-operator-prometheus 9091:9090 + ``` You also probably want to start with the argument `--useOutClusterClient=false`. When `useOutClusterClient` is `false`, several debugging settings are enabled: * a service principal with superpowers is used to authorize the requests, and the client's `Authorization` bearer token is ignored. @@ -52,6 +56,7 @@ You also probably want to start with the argument `--useOutClusterClient=false`. * the server CORS settings are modified to accept the `X-Requested-With` header in incoming requests. This is necessary to allow direct requests from web browser while e.g. debugging [radix-web-console](https://github.com/equinor/radix-web-console). * verbose debugging output from CORS rule evaluation is logged to console. + If you are using VSCode, there is a convenient launch configuration in `.vscode`. #### Validate code diff --git a/api/applications/applications_controller.go b/api/applications/applications_controller.go index 6ee6ada1..4afef764 100644 --- a/api/applications/applications_controller.go +++ b/api/applications/applications_controller.go @@ -8,6 +8,7 @@ import ( "strings" applicationModels "github.com/equinor/radix-api/api/applications/models" + "github.com/equinor/radix-api/api/metrics" "github.com/equinor/radix-api/models" "github.com/gorilla/mux" ) @@ -19,10 +20,11 @@ type applicationController struct { *models.DefaultController hasAccessToRR applicationHandlerFactory ApplicationHandlerFactory + prometheusHandler metrics.PrometheusHandler } // NewApplicationController Constructor -func NewApplicationController(hasAccessTo hasAccessToRR, applicationHandlerFactory ApplicationHandlerFactory) models.Controller { +func NewApplicationController(hasAccessTo hasAccessToRR, applicationHandlerFactory ApplicationHandlerFactory, prometheusHandler metrics.PrometheusHandler) models.Controller { if hasAccessTo == nil { hasAccessTo = hasAccess } @@ -30,6 +32,7 @@ func NewApplicationController(hasAccessTo hasAccessToRR, applicationHandlerFacto return &applicationController{ hasAccessToRR: hasAccessTo, applicationHandlerFactory: applicationHandlerFactory, + prometheusHandler: prometheusHandler, } } @@ -133,6 +136,11 @@ func (ac *applicationController) GetRoutes() models.Routes { Method: "POST", HandlerFunc: ac.RegenerateDeployKeyHandler, }, + models.Route{ + Path: appPath + "/resources", + Method: "GET", + HandlerFunc: ac.GetUsedResources, + }, } return routes @@ -992,3 +1000,66 @@ func (ac *applicationController) TriggerPipelinePromote(accounts models.Accounts ac.JSONResponse(w, r, &jobSummary) } + +// GetUsedResources Gets used resources for the application +func (ac *applicationController) GetUsedResources(accounts models.Accounts, w http.ResponseWriter, r *http.Request) { + // swagger:operation GET /applications/{appName}/resources application getResources + // --- + // summary: Gets used resources for the application + // parameters: + // - name: appName + // in: path + // description: Name of the application + // type: string + // required: true + // - name: environment + // in: query + // description: Name of the application environment + // type: string + // required: false + // - name: component + // in: query + // description: Name of the application component in an environment + // type: string + // required: false + // - name: duration + // in: query + // description: Duration of the period, default is 30d (30 days). Example 10m, 1h, 2d, 3w, where m-minutes, h-hours, d-days, w-weeks + // type: string + // required: false + // - name: since + // in: query + // description: End time-point of the period in the past, default is now. Example 10m, 1h, 2d, 3w, where m-minutes, h-hours, d-days, w-weeks + // type: string + // required: false + // - name: Impersonate-User + // in: header + // description: Works only with custom setup of cluster. Allow impersonation of test users (Required if Impersonate-Group is set) + // type: string + // required: false + // - name: Impersonate-Group + // in: header + // description: Works only with custom setup of cluster. Allow impersonation of a comma-seperated list of test groups (Required if Impersonate-User is set) + // type: string + // required: false + // responses: + // "200": + // description: Successful trigger pipeline + // schema: + // "$ref": "#/definitions/UsedResources" + // "404": + // description: "Not found" + appName := mux.Vars(r)["appName"] + envName := r.FormValue("environment") + componentName := r.FormValue("component") + duration := r.FormValue("duration") + since := r.FormValue("since") + + usedResources, err := ac.prometheusHandler.GetUsedResources(r.Context(), accounts.UserAccount.RadixClient, appName, envName, componentName, duration, since) + if err != nil { + ac.ErrorResponse(w, r, err) + return + } + + ac.JSONResponse(w, r, &usedResources) +} diff --git a/api/applications/applications_controller_test.go b/api/applications/applications_controller_test.go index 0e9f5e20..b09a864d 100644 --- a/api/applications/applications_controller_test.go +++ b/api/applications/applications_controller_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "net/http" "net/url" @@ -16,6 +17,7 @@ import ( applicationModels "github.com/equinor/radix-api/api/applications/models" environmentModels "github.com/equinor/radix-api/api/environments/models" jobModels "github.com/equinor/radix-api/api/jobs/models" + metricsMock "github.com/equinor/radix-api/api/metrics/mock" controllertest "github.com/equinor/radix-api/api/test" "github.com/equinor/radix-api/api/utils" "github.com/equinor/radix-api/models" @@ -32,6 +34,7 @@ import ( commontest "github.com/equinor/radix-operator/pkg/apis/test" builders "github.com/equinor/radix-operator/pkg/apis/utils" radixfake "github.com/equinor/radix-operator/pkg/client/clientset/versioned/fake" + "github.com/golang/mock/gomock" "github.com/google/uuid" kedafake "github.com/kedacore/keda/v2/pkg/generated/clientset/versioned/fake" prometheusfake "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/fake" @@ -73,6 +76,7 @@ func setupTestWithFactory(t *testing.T, handlerFactory ApplicationHandlerFactory err := commonTestUtils.CreateClusterPrerequisites(clusterName, egressIps, subscriptionId) require.NoError(t, err) _ = os.Setenv(defaults.ActiveClusternameEnvironmentVariable, clusterName) + prometheusHandlerMock := createPrometheusHandlerMock(t, radixclient, nil) // controllerTestUtils is used for issuing HTTP request and processing responses controllerTestUtils := controllertest.NewTestUtils( @@ -81,17 +85,25 @@ func setupTestWithFactory(t *testing.T, handlerFactory ApplicationHandlerFactory kedaClient, secretproviderclient, certClient, - NewApplicationController( - func(_ context.Context, _ kubernetes.Interface, _ v1.RadixRegistration) (bool, error) { - return true, nil - }, - handlerFactory, - ), + NewApplicationController(func(_ context.Context, _ kubernetes.Interface, _ v1.RadixRegistration) (bool, error) { + return true, nil + }, handlerFactory, prometheusHandlerMock), ) return &commonTestUtils, &controllerTestUtils, kubeclient, radixclient, kedaClient, prometheusclient, secretproviderclient, certClient } +func createPrometheusHandlerMock(t *testing.T, radixclient *radixfake.Clientset, mockHandler *func(handler *metricsMock.MockPrometheusHandler)) *metricsMock.MockPrometheusHandler { + ctrl := gomock.NewController(t) + mockPrometheusHandler := metricsMock.NewMockPrometheusHandler(ctrl) + if mockHandler != nil { + (*mockHandler)(mockPrometheusHandler) + } else { + mockPrometheusHandler.EXPECT().GetUsedResources(gomock.Any(), radixclient, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(&applicationModels.UsedResources{}, nil) + } + return mockPrometheusHandler +} + func TestGetApplications_HasAccessToSomeRR(t *testing.T) { commonTestUtils, _, kubeclient, radixclient, kedaClient, _, secretproviderclient, certClient := setupTest(t, true, true) @@ -103,19 +115,19 @@ func TestGetApplications_HasAccessToSomeRR(t *testing.T) { require.NoError(t, err) t.Run("no access", func(t *testing.T) { + prometheusHandlerMock := createPrometheusHandlerMock(t, radixclient, nil) controllerTestUtils := controllertest.NewTestUtils( kubeclient, radixclient, kedaClient, secretproviderclient, certClient, - NewApplicationController( - func(_ context.Context, _ kubernetes.Interface, _ v1.RadixRegistration) (bool, error) { - return false, nil - }, newTestApplicationHandlerFactory(ApplicationHandlerConfig{RequireAppConfigurationItem: true, RequireAppADGroups: true}, - func(ctx context.Context, kubeClient kubernetes.Interface, namespace string, configMapName string) (bool, error) { - return true, nil - }))) + NewApplicationController(func(_ context.Context, _ kubernetes.Interface, _ v1.RadixRegistration) (bool, error) { + return false, nil + }, newTestApplicationHandlerFactory(ApplicationHandlerConfig{RequireAppConfigurationItem: true, RequireAppADGroups: true}, + func(ctx context.Context, kubeClient kubernetes.Interface, namespace string, configMapName string) (bool, error) { + return true, nil + }), prometheusHandlerMock)) responseChannel := controllerTestUtils.ExecuteRequest("GET", "/api/v1/applications") response := <-responseChannel @@ -126,13 +138,13 @@ func TestGetApplications_HasAccessToSomeRR(t *testing.T) { }) t.Run("access to single app", func(t *testing.T) { - controllerTestUtils := controllertest.NewTestUtils(kubeclient, radixclient, kedaClient, secretproviderclient, certClient, NewApplicationController( - func(_ context.Context, _ kubernetes.Interface, rr v1.RadixRegistration) (bool, error) { - return rr.GetName() == "my-second-app", nil - }, newTestApplicationHandlerFactory(ApplicationHandlerConfig{RequireAppConfigurationItem: true, RequireAppADGroups: true}, - func(ctx context.Context, kubeClient kubernetes.Interface, namespace string, configMapName string) (bool, error) { - return true, nil - }))) + prometheusHandlerMock := createPrometheusHandlerMock(t, radixclient, nil) + controllerTestUtils := controllertest.NewTestUtils(kubeclient, radixclient, kedaClient, secretproviderclient, certClient, NewApplicationController(func(_ context.Context, _ kubernetes.Interface, rr v1.RadixRegistration) (bool, error) { + return rr.GetName() == "my-second-app", nil + }, newTestApplicationHandlerFactory(ApplicationHandlerConfig{RequireAppConfigurationItem: true, RequireAppADGroups: true}, + func(ctx context.Context, kubeClient kubernetes.Interface, namespace string, configMapName string) (bool, error) { + return true, nil + }), prometheusHandlerMock)) responseChannel := controllerTestUtils.ExecuteRequest("GET", "/api/v1/applications") response := <-responseChannel @@ -143,13 +155,13 @@ func TestGetApplications_HasAccessToSomeRR(t *testing.T) { }) t.Run("access to all app", func(t *testing.T) { - controllerTestUtils := controllertest.NewTestUtils(kubeclient, radixclient, kedaClient, secretproviderclient, certClient, NewApplicationController( - func(_ context.Context, _ kubernetes.Interface, _ v1.RadixRegistration) (bool, error) { + prometheusHandlerMock := createPrometheusHandlerMock(t, radixclient, nil) + controllerTestUtils := controllertest.NewTestUtils(kubeclient, radixclient, kedaClient, secretproviderclient, certClient, NewApplicationController(func(_ context.Context, _ kubernetes.Interface, _ v1.RadixRegistration) (bool, error) { + return true, nil + }, newTestApplicationHandlerFactory(ApplicationHandlerConfig{RequireAppConfigurationItem: true, RequireAppADGroups: true}, + func(ctx context.Context, kubeClient kubernetes.Interface, namespace string, configMapName string) (bool, error) { return true, nil - }, newTestApplicationHandlerFactory(ApplicationHandlerConfig{RequireAppConfigurationItem: true, RequireAppADGroups: true}, - func(ctx context.Context, kubeClient kubernetes.Interface, namespace string, configMapName string) (bool, error) { - return true, nil - }))) + }), prometheusHandlerMock)) responseChannel := controllerTestUtils.ExecuteRequest("GET", "/api/v1/applications") response := <-responseChannel @@ -178,7 +190,7 @@ func TestGetApplications_WithFilterOnSSHRepo_Filter(t *testing.T) { assert.Equal(t, 1, len(applications)) }) - t.Run("unmatching repo", func(t *testing.T) { + t.Run("not matching repo", func(t *testing.T) { responseChannel := controllerTestUtils.ExecuteRequest("GET", fmt.Sprintf("/api/v1/applications?sshRepo=%s", url.QueryEscape("git@github.com:Equinor/my-app2.git"))) response := <-responseChannel @@ -224,13 +236,13 @@ func TestSearchApplicationsPost(t *testing.T) { ) require.NoError(t, err) - controllerTestUtils := controllertest.NewTestUtils(kubeclient, radixclient, kedaClient, secretproviderclient, certClient, NewApplicationController( - func(_ context.Context, _ kubernetes.Interface, _ v1.RadixRegistration) (bool, error) { + prometheusHandlerMock := createPrometheusHandlerMock(t, radixclient, nil) + controllerTestUtils := controllertest.NewTestUtils(kubeclient, radixclient, kedaClient, secretproviderclient, certClient, NewApplicationController(func(_ context.Context, _ kubernetes.Interface, _ v1.RadixRegistration) (bool, error) { + return true, nil + }, newTestApplicationHandlerFactory(ApplicationHandlerConfig{RequireAppConfigurationItem: true, RequireAppADGroups: true}, + func(ctx context.Context, kubeClient kubernetes.Interface, namespace string, configMapName string) (bool, error) { return true, nil - }, newTestApplicationHandlerFactory(ApplicationHandlerConfig{RequireAppConfigurationItem: true, RequireAppADGroups: true}, - func(ctx context.Context, kubeClient kubernetes.Interface, namespace string, configMapName string) (bool, error) { - return true, nil - }))) + }), prometheusHandlerMock)) // Tests t.Run("search for "+appNames[0], func(t *testing.T) { @@ -306,13 +318,13 @@ func TestSearchApplicationsPost(t *testing.T) { }) t.Run("search for "+appNames[0]+" - no access", func(t *testing.T) { - controllerTestUtils := controllertest.NewTestUtils(kubeclient, radixclient, kedaClient, secretproviderclient, certClient, NewApplicationController( - func(_ context.Context, _ kubernetes.Interface, _ v1.RadixRegistration) (bool, error) { - return false, nil - }, newTestApplicationHandlerFactory(ApplicationHandlerConfig{RequireAppConfigurationItem: true, RequireAppADGroups: true}, - func(ctx context.Context, kubeClient kubernetes.Interface, namespace string, configMapName string) (bool, error) { - return true, nil - }))) + prometheusHandlerMock := createPrometheusHandlerMock(t, radixclient, nil) + controllerTestUtils := controllertest.NewTestUtils(kubeclient, radixclient, kedaClient, secretproviderclient, certClient, NewApplicationController(func(_ context.Context, _ kubernetes.Interface, _ v1.RadixRegistration) (bool, error) { + return false, nil + }, newTestApplicationHandlerFactory(ApplicationHandlerConfig{RequireAppConfigurationItem: true, RequireAppADGroups: true}, + func(ctx context.Context, kubeClient kubernetes.Interface, namespace string, configMapName string) (bool, error) { + return true, nil + }), prometheusHandlerMock)) params := applicationModels.ApplicationsSearchRequest{Names: []string{appNames[0]}} responseChannel := controllerTestUtils.ExecuteRequestWithParameters("POST", "/api/v1/applications/_search", ¶ms) response := <-responseChannel @@ -402,13 +414,13 @@ func TestSearchApplicationsGet(t *testing.T) { ) require.NoError(t, err) - controllerTestUtils := controllertest.NewTestUtils(kubeclient, radixclient, kedaClient, secretproviderclient, certClient, NewApplicationController( - func(_ context.Context, _ kubernetes.Interface, _ v1.RadixRegistration) (bool, error) { + prometheusHandlerMock := createPrometheusHandlerMock(t, radixclient, nil) + controllerTestUtils := controllertest.NewTestUtils(kubeclient, radixclient, kedaClient, secretproviderclient, certClient, NewApplicationController(func(_ context.Context, _ kubernetes.Interface, _ v1.RadixRegistration) (bool, error) { + return true, nil + }, newTestApplicationHandlerFactory(ApplicationHandlerConfig{RequireAppConfigurationItem: true, RequireAppADGroups: true}, + func(ctx context.Context, kubeClient kubernetes.Interface, namespace string, configMapName string) (bool, error) { return true, nil - }, newTestApplicationHandlerFactory(ApplicationHandlerConfig{RequireAppConfigurationItem: true, RequireAppADGroups: true}, - func(ctx context.Context, kubeClient kubernetes.Interface, namespace string, configMapName string) (bool, error) { - return true, nil - }))) + }), prometheusHandlerMock)) // Tests t.Run("search for "+appNames[0], func(t *testing.T) { @@ -474,13 +486,13 @@ func TestSearchApplicationsGet(t *testing.T) { }) t.Run("search for "+appNames[0]+" - no access", func(t *testing.T) { - controllerTestUtils := controllertest.NewTestUtils(kubeclient, radixclient, kedaClient, secretproviderclient, certClient, NewApplicationController( - func(_ context.Context, _ kubernetes.Interface, _ v1.RadixRegistration) (bool, error) { - return false, nil - }, newTestApplicationHandlerFactory(ApplicationHandlerConfig{RequireAppConfigurationItem: true, RequireAppADGroups: true}, - func(ctx context.Context, kubeClient kubernetes.Interface, namespace string, configMapName string) (bool, error) { - return true, nil - }))) + prometheusHandlerMock := createPrometheusHandlerMock(t, radixclient, nil) + controllerTestUtils := controllertest.NewTestUtils(kubeclient, radixclient, kedaClient, secretproviderclient, certClient, NewApplicationController(func(_ context.Context, _ kubernetes.Interface, _ v1.RadixRegistration) (bool, error) { + return false, nil + }, newTestApplicationHandlerFactory(ApplicationHandlerConfig{RequireAppConfigurationItem: true, RequireAppADGroups: true}, + func(ctx context.Context, kubeClient kubernetes.Interface, namespace string, configMapName string) (bool, error) { + return true, nil + }), prometheusHandlerMock)) params := "apps=" + appNames[0] responseChannel := controllerTestUtils.ExecuteRequest("GET", "/api/v1/applications/_search?"+params) response := <-responseChannel @@ -1764,8 +1776,8 @@ func TestDeleteApplication_ApplicationIsDeleted(t *testing.T) { func TestGetApplication_WithAppAlias_ContainsAppAlias(t *testing.T) { // Setup - commonTestUtils, controllerTestUtils, client, radixclient, kedaClient, promclient, secretproviderclient, certClient := setupTest(t, true, true) - err := utils.ApplyDeploymentWithSync(client, radixclient, kedaClient, promclient, commonTestUtils, secretproviderclient, certClient, builders.ARadixDeployment(). + commonTestUtils, controllerTestUtils, client, radixclient, kedaClient, promClient, secretproviderclient, certClient := setupTest(t, true, true) + err := utils.ApplyDeploymentWithSync(client, radixclient, kedaClient, promClient, commonTestUtils, secretproviderclient, certClient, builders.ARadixDeployment(). WithAppName("any-app"). WithEnvironment("prod"). WithComponents( @@ -1792,7 +1804,7 @@ func TestGetApplication_WithAppAlias_ContainsAppAlias(t *testing.T) { assert.Equal(t, fmt.Sprintf("%s.%s", "any-app", appAliasDNSZone), application.AppAlias.URL) } -func TestListPipeline_ReturnesAvailablePipelines(t *testing.T) { +func TestListPipeline_ReturnsAvailablePipelines(t *testing.T) { supportedPipelines := jobPipeline.GetSupportedPipelines() // Setup @@ -1921,6 +1933,109 @@ func TestRegenerateDeployKey_InvalidKeyInParam_ErrorIsReturned(t *testing.T) { assert.Equal(t, http.StatusBadRequest, response.Code) } +func Test_GetUsedResources(t *testing.T) { + const ( + appName1 = "app-1" + envName1 = "prod" + componentName1 = "component1" + ) + + type expectedArgs struct { + environment string + component string + duration string + since string + } + + type scenario struct { + name string + expectedUsedResources *applicationModels.UsedResources + expectedError error + queryString string + expectedArgs expectedArgs + expectedUsedResourcesError error + } + + scenarios := []scenario{ + { + name: "Get used resources", + expectedUsedResources: getTestUsedResources(), + expectedArgs: expectedArgs{}, + }, + { + name: "Get used resources with arguments", + queryString: "?environment=prod&component=component1&duration=10d&since=2w", + expectedUsedResources: getTestUsedResources(), + expectedArgs: expectedArgs{ + environment: envName1, + component: componentName1, + duration: "10d", + since: "2w", + }, + }, + { + name: "UsedResources returns an error", + expectedUsedResources: getTestUsedResources(), + expectedUsedResourcesError: errors.New("error-123"), + expectedError: errors.New("Error: error-123"), + }, + } + + for _, ts := range scenarios { + t.Run(ts.name, func(t *testing.T) { + commonTestUtils, _, kubeClient, radixClient, kedaClient, _, secretProviderClient, certClient := setupTest(t, true, true) + _, err := commonTestUtils.ApplyRegistration(builders.ARadixRegistration().WithName(appName1)) + require.NoError(t, err) + + mockHandlerModifier := func(handler *metricsMock.MockPrometheusHandler) { + args := ts.expectedArgs + handler.EXPECT().GetUsedResources(gomock.Any(), radixClient, appName1, args.environment, args.component, args.duration, args.since). + Times(1). + Return(ts.expectedUsedResources, ts.expectedUsedResourcesError) + } + prometheusHandlerMock := createPrometheusHandlerMock(t, radixClient, &mockHandlerModifier) + controllerTestUtils := controllertest.NewTestUtils(kubeClient, radixClient, kedaClient, secretProviderClient, certClient, NewApplicationController(func(_ context.Context, _ kubernetes.Interface, _ v1.RadixRegistration) (bool, error) { + return true, nil + }, newTestApplicationHandlerFactory(ApplicationHandlerConfig{RequireAppConfigurationItem: true, RequireAppADGroups: true}, + func(ctx context.Context, kubeClient kubernetes.Interface, namespace string, configMapName string) (bool, error) { + return true, nil + }), prometheusHandlerMock)) + + responseChannel := controllerTestUtils.ExecuteRequest("GET", fmt.Sprintf("/api/v1/applications/%s/resources%s", appName1, ts.queryString)) + response := <-responseChannel + if ts.expectedError != nil { + assert.Equal(t, http.StatusBadRequest, response.Code) + errorResponse, _ := controllertest.GetErrorResponse(response) + assert.Equal(t, ts.expectedError.Error(), errorResponse.Message) + return + } + assert.Equal(t, http.StatusOK, response.Code) + actualUsedResources := &applicationModels.UsedResources{} + err = controllertest.GetResponseBody(response, &actualUsedResources) + require.NoError(t, err) + assert.Equal(t, ts.expectedUsedResources, actualUsedResources) + }) + } +} + +func getTestUsedResources() *applicationModels.UsedResources { + return &applicationModels.UsedResources{ + From: radixutils.FormatTimestamp(time.Now().Add(time.Minute * -10)), + To: radixutils.FormatTimestamp(time.Now()), + CPU: &applicationModels.UsedResource{ + Min: pointers.Ptr(1.1), + Max: pointers.Ptr(10.12), + Avg: pointers.Ptr(5.56), + }, + Memory: &applicationModels.UsedResource{ + Min: pointers.Ptr(100.1), + Max: pointers.Ptr(1000.12), + Avg: pointers.Ptr(500.56), + }, + Warnings: []string{"warning1", "warning2"}, + } +} + func createRadixJob(commonTestUtils *commontest.Utils, appName, jobName string, started time.Time) error { _, err := commonTestUtils.ApplyJob( builders.ARadixBuildDeployJob(). diff --git a/api/applications/applications_handler_config.go b/api/applications/applications_handler_config.go index 87c749a2..ae190e54 100644 --- a/api/applications/applications_handler_config.go +++ b/api/applications/applications_handler_config.go @@ -32,6 +32,7 @@ type ApplicationHandlerConfig struct { AppName string `cfg:"radix_app" flag:"radix-app"` EnvironmentName string `cfg:"radix_environment" flag:"radix-environment"` DNSZone string `cfg:"radix_dns_zone" flag:"radix-dns-zone"` + PrometheusUrl string `cfg:"prometheus_url" flag:"prometheus-url"` } func ApplicationHandlerConfigFlagSet() *pflag.FlagSet { @@ -42,5 +43,6 @@ func ApplicationHandlerConfigFlagSet() *pflag.FlagSet { flagset.String("radix-app", "", "Application name") flagset.String("radix-environment", "", "Environment name") flagset.String("radix-dns-zone", "", "Radix DNS zone") + flagset.String("prometheus-url", "", "Prometheus URL") return flagset } diff --git a/api/applications/models/used_resources.go b/api/applications/models/used_resources.go new file mode 100644 index 00000000..4c89eeed --- /dev/null +++ b/api/applications/models/used_resources.go @@ -0,0 +1,54 @@ +package models + +// UsedResources holds information about used resources +// swagger:model UsedResources +type UsedResources struct { + // From timestamp + // + // required: true + // example: 2006-01-02T15:04:05Z + From string `json:"from"` + + // To timestamp + // + // required: true + // example: 2006-01-03T15:04:05Z + To string `json:"to"` + + // CPU used, in cores + // + // required: false + CPU *UsedResource `json:"cpu,omitempty"` + + // Memory used, in bytes + // + // required: false + Memory *UsedResource `json:"memory,omitempty"` + + // Warning messages + // + // required: false + Warnings []string `json:"warnings,omitempty"` +} + +// UsedResource holds information about used resource +// swagger:model UsedResource +type UsedResource struct { + // Min resource used + // + // required: false + // example: 0.00012 + Min *float64 `json:"min,omitempty"` + + // Avg Average resource used + // + // required: false + // example: 0.00023 + Avg *float64 `json:"avg,omitempty"` + + // Max resource used + // + // required: false + // example: 0.00037 + Max *float64 `json:"max,omitempty"` +} diff --git a/api/metrics/internal/prometheus_defaults.go b/api/metrics/internal/prometheus_defaults.go new file mode 100644 index 00000000..f284370b --- /dev/null +++ b/api/metrics/internal/prometheus_defaults.go @@ -0,0 +1,13 @@ +package internal + +// QueryName Prometheus query name +type QueryName string + +const ( + CpuMax QueryName = "CpuMax" + CpuMin QueryName = "CpuMin" + CpuAvg QueryName = "CpuAvg" + MemoryMax QueryName = "MemoryMax" + MemoryMin QueryName = "MemoryMin" + MemoryAvg QueryName = "MemoryAvg" +) diff --git a/api/metrics/mock/prometheus_client_mock.go b/api/metrics/mock/prometheus_client_mock.go new file mode 100644 index 00000000..c42ab46e --- /dev/null +++ b/api/metrics/mock/prometheus_client_mock.go @@ -0,0 +1,53 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./api/metrics/prometheus_client.go + +// Package mock is a generated GoMock package. +package mock + +import ( + context "context" + reflect "reflect" + + internal "github.com/equinor/radix-api/api/metrics/internal" + gomock "github.com/golang/mock/gomock" + model "github.com/prometheus/common/model" +) + +// MockPrometheusClient is a mock of PrometheusClient interface. +type MockPrometheusClient struct { + ctrl *gomock.Controller + recorder *MockPrometheusClientMockRecorder +} + +// MockPrometheusClientMockRecorder is the mock recorder for MockPrometheusClient. +type MockPrometheusClientMockRecorder struct { + mock *MockPrometheusClient +} + +// NewMockPrometheusClient creates a new mock instance. +func NewMockPrometheusClient(ctrl *gomock.Controller) *MockPrometheusClient { + mock := &MockPrometheusClient{ctrl: ctrl} + mock.recorder = &MockPrometheusClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockPrometheusClient) EXPECT() *MockPrometheusClientMockRecorder { + return m.recorder +} + +// GetMetrics mocks base method. +func (m *MockPrometheusClient) GetMetrics(ctx context.Context, appName, envName, componentName, duration, since string) (map[internal.QueryName]model.Value, []string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMetrics", ctx, appName, envName, componentName, duration, since) + ret0, _ := ret[0].(map[internal.QueryName]model.Value) + ret1, _ := ret[1].([]string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetMetrics indicates an expected call of GetMetrics. +func (mr *MockPrometheusClientMockRecorder) GetMetrics(ctx, appName, envName, componentName, duration, since interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMetrics", reflect.TypeOf((*MockPrometheusClient)(nil).GetMetrics), ctx, appName, envName, componentName, duration, since) +} diff --git a/api/metrics/mock/prometheus_handler_mock.go b/api/metrics/mock/prometheus_handler_mock.go new file mode 100644 index 00000000..255cbe2b --- /dev/null +++ b/api/metrics/mock/prometheus_handler_mock.go @@ -0,0 +1,52 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./api/metrics/prometheus_handler.go + +// Package mock is a generated GoMock package. +package mock + +import ( + context "context" + reflect "reflect" + + models "github.com/equinor/radix-api/api/applications/models" + versioned "github.com/equinor/radix-operator/pkg/client/clientset/versioned" + gomock "github.com/golang/mock/gomock" +) + +// MockPrometheusHandler is a mock of PrometheusHandler interface. +type MockPrometheusHandler struct { + ctrl *gomock.Controller + recorder *MockPrometheusHandlerMockRecorder +} + +// MockPrometheusHandlerMockRecorder is the mock recorder for MockPrometheusHandler. +type MockPrometheusHandlerMockRecorder struct { + mock *MockPrometheusHandler +} + +// NewMockPrometheusHandler creates a new mock instance. +func NewMockPrometheusHandler(ctrl *gomock.Controller) *MockPrometheusHandler { + mock := &MockPrometheusHandler{ctrl: ctrl} + mock.recorder = &MockPrometheusHandlerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockPrometheusHandler) EXPECT() *MockPrometheusHandlerMockRecorder { + return m.recorder +} + +// GetUsedResources mocks base method. +func (m *MockPrometheusHandler) GetUsedResources(ctx context.Context, radixClient versioned.Interface, appName, envName, componentName, duration, since string) (*models.UsedResources, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetUsedResources", ctx, radixClient, appName, envName, componentName, duration, since) + ret0, _ := ret[0].(*models.UsedResources) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetUsedResources indicates an expected call of GetUsedResources. +func (mr *MockPrometheusHandlerMockRecorder) GetUsedResources(ctx, radixClient, appName, envName, componentName, duration, since interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUsedResources", reflect.TypeOf((*MockPrometheusHandler)(nil).GetUsedResources), ctx, radixClient, appName, envName, componentName, duration, since) +} diff --git a/api/metrics/prometheus_client.go b/api/metrics/prometheus_client.go new file mode 100644 index 00000000..718a15e5 --- /dev/null +++ b/api/metrics/prometheus_client.go @@ -0,0 +1,78 @@ +package metrics + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/equinor/radix-api/api/metrics/internal" + radixutils "github.com/equinor/radix-common/utils" + "github.com/equinor/radix-operator/pkg/apis/utils" + prometheusApi "github.com/prometheus/client_golang/api" + prometheusV1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" + prometheusModel "github.com/prometheus/common/model" + "github.com/rs/zerolog/log" +) + +// PrometheusClient Interface for Prometheus client +type PrometheusClient interface { + // GetMetrics Get metrics for the application + GetMetrics(ctx context.Context, appName, envName, componentName, duration, since string) (map[internal.QueryName]prometheusModel.Value, []string, error) +} + +// NewPrometheusClient Constructor for Prometheus client +func NewPrometheusClient(prometheusUrl string) (PrometheusClient, error) { + apiClient, err := prometheusApi.NewClient(prometheusApi.Config{Address: prometheusUrl}) + if err != nil { + return nil, errors.New("failed to create the Prometheus API client") + } + api := prometheusV1.NewAPI(apiClient) + return &client{ + api: api, + }, nil +} + +type client struct { + api prometheusV1.API +} + +// GetMetrics Get metrics for the application +func (c *client) GetMetrics(ctx context.Context, appName, envName, componentName, duration, since string) (map[internal.QueryName]prometheusModel.Value, []string, error) { + results := make(map[internal.QueryName]model.Value) + now := time.Now() + var warnings []string + for metricName, query := range getPrometheusQueries(appName, envName, componentName, duration, since) { + result, resultWarnings, err := c.api.Query(ctx, query, now) + if err != nil { + log.Ctx(ctx).Error().Msgf("Failed to get Prometheus metrics: %v", err) + return nil, nil, errors.New("failed to get Prometheus metrics") + } + if len(resultWarnings) > 0 { + log.Ctx(ctx).Warn().Msgf("Warnings: %v\n", resultWarnings) + warnings = append(warnings, resultWarnings...) + } + results[metricName] = result + } + return results, warnings, nil +} + +func getPrometheusQueries(appName, envName, componentName, duration, since string) map[internal.QueryName]string { + environmentFilter := radixutils.TernaryString(envName == "", + fmt.Sprintf(`,namespace=~"%s-.*"`, appName), + fmt.Sprintf(`,namespace="%s"`, utils.GetEnvironmentNamespace(appName, envName))) + componentFilter := radixutils.TernaryString(envName == "", "", fmt.Sprintf(`,container="%s"`, componentName)) + offsetFilter := radixutils.TernaryString(since == "", "", fmt.Sprintf(` offset %s `, since)) + cpuUsageQuery := fmt.Sprintf(`sum by (namespace, container) (rate(container_cpu_usage_seconds_total{container!="", namespace!="%s-app" %s %s} [1h])) [%s:] %s`, appName, environmentFilter, componentFilter, duration, offsetFilter) + memoryUsageQuery := fmt.Sprintf(`sum by (namespace, container) (container_memory_usage_bytes{container!="", namespace!="%s-app" %s %s} > 0) [%s:] %s`, appName, environmentFilter, componentFilter, duration, offsetFilter) + queries := map[internal.QueryName]string{ + internal.CpuMax: fmt.Sprintf("max_over_time(%s)", cpuUsageQuery), + internal.CpuMin: fmt.Sprintf("min_over_time(%s)", cpuUsageQuery), + internal.CpuAvg: fmt.Sprintf("avg_over_time(%s)", cpuUsageQuery), + internal.MemoryMax: fmt.Sprintf("max_over_time(%s)", memoryUsageQuery), + internal.MemoryMin: fmt.Sprintf("min_over_time(%s)", memoryUsageQuery), + internal.MemoryAvg: fmt.Sprintf("avg_over_time(%s)", memoryUsageQuery), + } + return queries +} diff --git a/api/metrics/prometheus_handler.go b/api/metrics/prometheus_handler.go new file mode 100644 index 00000000..09bcd992 --- /dev/null +++ b/api/metrics/prometheus_handler.go @@ -0,0 +1,127 @@ +package metrics + +import ( + "context" + "regexp" + "time" + + applicationModels "github.com/equinor/radix-api/api/applications/models" + "github.com/equinor/radix-api/api/metrics/internal" + radixutils "github.com/equinor/radix-common/utils" + "github.com/equinor/radix-common/utils/pointers" + radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned" + "github.com/prometheus/common/model" + prometheusModel "github.com/prometheus/common/model" + "github.com/rs/zerolog/log" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + durationExpression = `^[0-9]{1,5}[mhdw]$` + defaultDuration = "30d" + defaultOffset = "" +) + +// PrometheusHandler Interface for Prometheus handler +type PrometheusHandler interface { + GetUsedResources(ctx context.Context, radixClient radixclient.Interface, appName, envName, componentName, duration, since string) (*applicationModels.UsedResources, error) +} + +type handler struct { + client PrometheusClient +} + +// NewPrometheusHandler Constructor for Prometheus handler +func NewPrometheusHandler(client PrometheusClient) PrometheusHandler { + return &handler{ + client: client, + } +} + +// GetUsedResources Get used resources for the application +func (pc *handler) GetUsedResources(ctx context.Context, radixClient radixclient.Interface, appName, envName, componentName, duration, since string) (*applicationModels.UsedResources, error) { + _, err := radixClient.RadixV1().RadixRegistrations().Get(ctx, appName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + durationValue, duration, err := parseQueryDuration(duration, defaultDuration) + if err != nil { + return nil, err + } + sinceValue, since, err := parseQueryDuration(since, defaultOffset) + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + log.Ctx(ctx).Debug().Msgf("Getting used resources for application %s", appName) + results, warnings, err := pc.client.GetMetrics(ctx, appName, envName, componentName, duration, since) + if err != nil { + return nil, err + } + resources := getUsedResourcesByMetrics(ctx, results, durationValue, sinceValue) + resources.Warnings = warnings + log.Ctx(ctx).Debug().Msgf("Got used resources for application %s", appName) + return resources, nil +} + +func getUsedResourcesByMetrics(ctx context.Context, results map[internal.QueryName]prometheusModel.Value, queryDuration time.Duration, querySince time.Duration) *applicationModels.UsedResources { + usedCpuResource := applicationModels.UsedResource{} + usedCpuResource.Min = getCpuMetricValue(ctx, results, internal.CpuMin) + usedCpuResource.Max = getCpuMetricValue(ctx, results, internal.CpuMax) + usedCpuResource.Avg = getCpuMetricValue(ctx, results, internal.CpuAvg) + usedMemoryResource := applicationModels.UsedResource{} + usedMemoryResource.Min = getMemoryMetricValue(ctx, results, internal.MemoryMin) + usedMemoryResource.Max = getMemoryMetricValue(ctx, results, internal.MemoryMax) + usedMemoryResource.Avg = getMemoryMetricValue(ctx, results, internal.MemoryAvg) + now := time.Now() + return &applicationModels.UsedResources{ + From: radixutils.FormatTimestamp(now.Add(-queryDuration)), + To: radixutils.FormatTimestamp(now.Add(-querySince)), + CPU: &usedCpuResource, + Memory: &usedMemoryResource, + } +} + +func parseQueryDuration(duration string, defaultValue string) (time.Duration, string, error) { + if len(duration) == 0 || !regexp.MustCompile(durationExpression).MatchString(duration) { + duration = defaultValue + } + if len(duration) == 0 { + return 0, duration, nil + } + parsedDuration, err := prometheusModel.ParseDuration(duration) + return time.Duration(parsedDuration), duration, err +} + +func getCpuMetricValue(ctx context.Context, queryResults map[internal.QueryName]prometheusModel.Value, queryName internal.QueryName) *float64 { + if value, ok := getMetricsValue(ctx, queryResults, queryName); ok { + return pointers.Ptr(value) + } + return nil +} + +func getMemoryMetricValue(ctx context.Context, queryResults map[internal.QueryName]prometheusModel.Value, queryName internal.QueryName) *float64 { + if value, ok := getMetricsValue(ctx, queryResults, queryName); ok { + return pointers.Ptr(value) + } + return nil +} + +func getMetricsValue(ctx context.Context, queryResults map[internal.QueryName]prometheusModel.Value, queryName internal.QueryName) (float64, bool) { + queryResult, ok := queryResults[queryName] + if !ok { + return 0.0, false + } + groupedMetrics, ok := queryResult.(model.Vector) + if !ok { + log.Ctx(ctx).Error().Msgf("Failed to convert metrics query %s result to Vector", queryName) + return 0, false + } + sum := 0.0 + for _, sample := range groupedMetrics { + sum += float64(sample.Value) + } + return sum, true +} diff --git a/api/metrics/prometheus_handler_test.go b/api/metrics/prometheus_handler_test.go new file mode 100644 index 00000000..beb8abdd --- /dev/null +++ b/api/metrics/prometheus_handler_test.go @@ -0,0 +1,173 @@ +package metrics + +import ( + "context" + "errors" + "testing" + + applicationModels "github.com/equinor/radix-api/api/applications/models" + "github.com/equinor/radix-api/api/metrics/internal" + "github.com/equinor/radix-api/api/metrics/mock" + "github.com/equinor/radix-common/utils/pointers" + commontest "github.com/equinor/radix-operator/pkg/apis/test" + builders "github.com/equinor/radix-operator/pkg/apis/utils" + "github.com/equinor/radix-operator/pkg/client/clientset/versioned/fake" + "github.com/golang/mock/gomock" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type args struct { + appName string + envName string + componentName string + duration string + since string +} + +type scenario struct { + name string + args args + clientReturnsMetrics map[internal.QueryName]model.Value + expectedUsedResources *applicationModels.UsedResources + expectedWarnings []string + expectedError error +} + +const ( + appName1 = "app1" + metricsKeyContainer = "container" + metricsKeyNamespace = "namespace" +) + +func Test_handler_GetUsedResources(t *testing.T) { + scenarios := []scenario{ + { + name: "Got used resources", + args: args{ + appName: appName1, + duration: defaultDuration, + }, + clientReturnsMetrics: getClientReturnsMetrics(), + expectedUsedResources: getExpectedUsedResources(), + }, + { + name: "Got used resources with warnings", + args: args{ + appName: appName1, + duration: defaultDuration, + }, + clientReturnsMetrics: getClientReturnsMetrics(), + expectedUsedResources: getExpectedUsedResources("Warning1", "Warning2"), + expectedWarnings: []string{"Warning1", "Warning2"}, + }, + { + name: "Requested with arguments", + args: args{ + appName: appName1, + envName: "dev", + componentName: "component1", + duration: defaultDuration, + since: "2d", + }, + clientReturnsMetrics: getClientReturnsMetrics(), + expectedUsedResources: getExpectedUsedResources(), + }, + { + name: "With error", + args: args{ + appName: appName1, + duration: defaultDuration, + }, + expectedError: errors.New("failed to get Prometheus metrics"), + }, + } + for _, ts := range scenarios { + t.Run(ts.name, func(t *testing.T) { + radixClient := fake.NewSimpleClientset() + commonTestUtils := commontest.NewTestUtils(nil, radixClient, nil, nil) + _, err := commonTestUtils.ApplyRegistration(builders.ARadixRegistration().WithName(appName1)) + require.NoError(t, err) + ctrl := gomock.NewController(t) + mockPrometheusClient := mock.NewMockPrometheusClient(ctrl) + mockPrometheusClient.EXPECT().GetMetrics(gomock.Any(), appName1, ts.args.envName, ts.args.componentName, ts.args.duration, ts.args.since). + Return(ts.clientReturnsMetrics, ts.expectedWarnings, ts.expectedError) + + prometheusHandler := &handler{ + client: mockPrometheusClient, + } + got, err := prometheusHandler.GetUsedResources(context.Background(), radixClient, appName1, ts.args.envName, ts.args.componentName, ts.args.duration, ts.args.since) + if ts.expectedError != nil { + assert.ErrorIs(t, err, ts.expectedError, "Missing or not matching GetUsedResources() error") + return + } else { + require.NoError(t, err, "Missing or not matching GetUsedResources() error") + } + assertExpected(t, ts, got) + }) + } +} + +func assertExpected(t *testing.T, ts scenario, got *applicationModels.UsedResources) { + assert.ElementsMatch(t, ts.expectedWarnings, got.Warnings, "Warnings") + assert.NotNil(t, got.CPU.Min, "nil CPU.Min") + assert.NotNil(t, got.CPU.Max, "nil CPU.Max") + assert.NotNil(t, *got.CPU.Avg, "nil CPU.Avg") + assert.Equal(t, *ts.expectedUsedResources.CPU.Min, *got.CPU.Min, "CPU.Min") + assert.Equal(t, *ts.expectedUsedResources.CPU.Max, *got.CPU.Max, "CPU.Max") + assert.Equal(t, *ts.expectedUsedResources.CPU.Avg, *got.CPU.Avg, "CPU.Avg") + assert.NotNil(t, got.Memory.Min, "nil Memory.Min") + assert.NotNil(t, got.Memory.Max, "nil Memory.Max") + assert.NotNil(t, got.Memory.Avg, "nil Memory.Avg") + assert.Equal(t, *ts.expectedUsedResources.Memory.Min, *got.Memory.Min, "Memory.Min") + assert.Equal(t, *ts.expectedUsedResources.Memory.Max, *got.Memory.Max, "Memory.Max") + assert.Equal(t, *ts.expectedUsedResources.Memory.Avg, *got.Memory.Avg, "Memory.Avg") + assert.NotEmpty(t, got.From, "From") + assert.NotEmpty(t, got.To, "To") +} + +func getClientReturnsMetrics() map[internal.QueryName]model.Value { + return map[internal.QueryName]model.Value{ + internal.CpuMax: model.Vector{ + &model.Sample{Metric: map[model.LabelName]model.LabelValue{metricsKeyContainer: "server", metricsKeyNamespace: "app-dev"}, Value: 0.008123134}, + &model.Sample{Metric: map[model.LabelName]model.LabelValue{metricsKeyContainer: "server", metricsKeyNamespace: "app-prod"}, Value: 0.126576764}, + }, + internal.CpuAvg: model.Vector{ + &model.Sample{Metric: map[model.LabelName]model.LabelValue{metricsKeyContainer: "server", metricsKeyNamespace: "app-dev"}, Value: 0.0023213546}, + &model.Sample{Metric: map[model.LabelName]model.LabelValue{metricsKeyContainer: "server", metricsKeyNamespace: "app-prod"}, Value: 0.047546577}, + }, + internal.CpuMin: model.Vector{ + &model.Sample{Metric: map[model.LabelName]model.LabelValue{metricsKeyContainer: "server", metricsKeyNamespace: "app-dev"}, Value: 0.0019874}, + &model.Sample{Metric: map[model.LabelName]model.LabelValue{metricsKeyContainer: "server", metricsKeyNamespace: "app-prod"}, Value: 0.02321456}, + }, + internal.MemoryMax: model.Vector{ + &model.Sample{Metric: map[model.LabelName]model.LabelValue{metricsKeyContainer: "server", metricsKeyNamespace: "app-dev"}, Value: 123456.3475613}, + &model.Sample{Metric: map[model.LabelName]model.LabelValue{metricsKeyContainer: "server", metricsKeyNamespace: "app-prod"}, Value: 234567.34575412}, + }, + internal.MemoryAvg: model.Vector{ + &model.Sample{Metric: map[model.LabelName]model.LabelValue{metricsKeyContainer: "server", metricsKeyNamespace: "app-dev"}, Value: 90654.81}, + &model.Sample{Metric: map[model.LabelName]model.LabelValue{metricsKeyContainer: "server", metricsKeyNamespace: "app-prod"}, Value: 150654.12398771}, + }, + internal.MemoryMin: model.Vector{ + &model.Sample{Metric: map[model.LabelName]model.LabelValue{metricsKeyContainer: "server", metricsKeyNamespace: "app-dev"}, Value: 56731.2324654}, + &model.Sample{Metric: map[model.LabelName]model.LabelValue{metricsKeyContainer: "server", metricsKeyNamespace: "app-prod"}, Value: 112234.456789}, + }, + } +} + +func getExpectedUsedResources(warnings ...string) *applicationModels.UsedResources { + return &applicationModels.UsedResources{ + Warnings: warnings, + CPU: &applicationModels.UsedResource{ + Min: pointers.Ptr(0.02520196), + Avg: pointers.Ptr(0.0498679316), + Max: pointers.Ptr(0.134699898), + }, + Memory: &applicationModels.UsedResource{ + Min: pointers.Ptr(168965.6892544), + Avg: pointers.Ptr(241308.93398770998), + Max: pointers.Ptr(358023.69331542), + }, + } +} diff --git a/go.mod b/go.mod index cf914047..e3b25aa0 100644 --- a/go.mod +++ b/go.mod @@ -13,13 +13,13 @@ require ( github.com/felixge/httpsnoop v1.0.4 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/golang/mock v1.6.0 - github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 github.com/kedacore/keda/v2 v2.15.1 github.com/marstr/guid v1.1.0 github.com/mitchellh/mapstructure v1.5.0 github.com/prometheus-operator/prometheus-operator/pkg/client v0.76.0 - github.com/prometheus/client_golang v1.20.2 + github.com/prometheus/client_golang v1.20.3 + github.com/prometheus/common v0.55.0 github.com/rs/cors v1.11.0 github.com/rs/xid v1.5.0 github.com/rs/zerolog v1.33.0 @@ -65,6 +65,7 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-containerregistry v0.16.1 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -86,7 +87,6 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.76.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/prometheus/statsd_exporter v0.22.7 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 008033ba..a026954f 100644 --- a/go.sum +++ b/go.sum @@ -233,6 +233,7 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -288,6 +289,7 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.20.0 h1:PE84V2mHqoT1sglvHc8ZdQtPcwmvvt29WLEEO3xmdZw= @@ -316,8 +318,8 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg= -github.com/prometheus/client_golang v1.20.2/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_golang v1.20.3 h1:oPksm4K8B+Vt35tUhw6GbSNSgVlVSBH0qELP/7u83l4= +github.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= diff --git a/main.go b/main.go index acca65de..e326148c 100644 --- a/main.go +++ b/main.go @@ -14,6 +14,7 @@ import ( "syscall" "time" + "github.com/equinor/radix-api/api/metrics" "github.com/equinor/radix-api/api/secrets" "github.com/equinor/radix-api/api/utils/tlsvalidation" "github.com/equinor/radix-operator/pkg/apis/defaults" @@ -168,13 +169,19 @@ func setupLogger() { func getControllers() ([]models.Controller, error) { buildStatus := build_models.NewPipelineBadge() - applicationHandlerFactory, err := getApplicationHandlerFactory() + cfg, err := applications.LoadApplicationHandlerConfig(os.Args[1:]) if err != nil { return nil, err } + prometheusClient, err := metrics.NewPrometheusClient(cfg.PrometheusUrl) + if err != nil { + return nil, err + } + prometheusHandler := metrics.NewPrometheusHandler(prometheusClient) + applicationHandlerFactory := applications.NewApplicationHandlerFactory(cfg) return []models.Controller{ - applications.NewApplicationController(nil, applicationHandlerFactory), + applications.NewApplicationController(nil, applicationHandlerFactory, prometheusHandler), deployments.NewDeploymentController(), jobs.NewJobController(), environments.NewEnvironmentController(environments.NewEnvironmentHandlerFactory()), @@ -187,14 +194,6 @@ func getControllers() ([]models.Controller, error) { }, nil } -func getApplicationHandlerFactory() (applications.ApplicationHandlerFactory, error) { - cfg, err := applications.LoadApplicationHandlerConfig(os.Args[1:]) - if err != nil { - return nil, err - } - return applications.NewApplicationHandlerFactory(cfg), nil -} - func initializeFlagSet() *pflag.FlagSet { // Flag domain. fs := pflag.NewFlagSet("default", pflag.ContinueOnError) diff --git a/radixconfig.yaml b/radixconfig.yaml index ee3b9454..91269a0a 100644 --- a/radixconfig.yaml +++ b/radixconfig.yaml @@ -34,6 +34,7 @@ spec: USE_PROFILER: "false" LOG_LEVEL: info LOG_PRETTY: "false" + PROMETHEUS_URL: http://prometheus-operator-prometheus.monitor.svc.cluster.local:9090 environmentConfig: - environment: qa horizontalScaling: diff --git a/swaggerui/html/swagger.json b/swaggerui/html/swagger.json index 1e0d9119..8ceb289f 100644 --- a/swaggerui/html/swagger.json +++ b/swaggerui/html/swagger.json @@ -5021,6 +5021,71 @@ } } }, + "/applications/{appName}/resources": { + "get": { + "tags": [ + "application" + ], + "summary": "Gets used resources for the application", + "operationId": "getResources", + "parameters": [ + { + "type": "string", + "description": "Name of the application", + "name": "appName", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Name of the application environment", + "name": "environment", + "in": "query" + }, + { + "type": "string", + "description": "Name of the application component in an environment", + "name": "component", + "in": "query" + }, + { + "type": "string", + "description": "Duration of the period, default is 30d (30 days). Example 10m, 1h, 2d, 3w, where m-minutes, h-hours, d-days, w-weeks", + "name": "duration", + "in": "query" + }, + { + "type": "string", + "description": "End time-point of the period in the past, default is now. Example 10m, 1h, 2d, 3w, where m-minutes, h-hours, d-days, w-weeks", + "name": "since", + "in": "query" + }, + { + "type": "string", + "description": "Works only with custom setup of cluster. Allow impersonation of test users (Required if Impersonate-Group is set)", + "name": "Impersonate-User", + "in": "header" + }, + { + "type": "string", + "description": "Works only with custom setup of cluster. Allow impersonation of a comma-seperated list of test groups (Required if Impersonate-User is set)", + "name": "Impersonate-Group", + "in": "header" + } + ], + "responses": { + "200": { + "description": "Successful trigger pipeline", + "schema": { + "$ref": "#/definitions/UsedResources" + } + }, + "404": { + "description": "Not found" + } + } + } + }, "/applications/{appName}/restart": { "post": { "tags": [ @@ -8239,6 +8304,71 @@ }, "x-go-package": "github.com/equinor/radix-api/api/alerting/models" }, + "UsedResource": { + "description": "UsedResource holds information about used resource", + "type": "object", + "properties": { + "avg": { + "description": "Avg Average resource used", + "type": "number", + "format": "double", + "x-go-name": "Avg", + "example": 0.00023 + }, + "max": { + "description": "Max resource used", + "type": "number", + "format": "double", + "x-go-name": "Max", + "example": 0.00037 + }, + "min": { + "description": "Min resource used", + "type": "number", + "format": "double", + "x-go-name": "Min", + "example": 0.00012 + } + }, + "x-go-package": "github.com/equinor/radix-api/api/applications/models" + }, + "UsedResources": { + "description": "UsedResources holds information about used resources", + "type": "object", + "required": [ + "from", + "to" + ], + "properties": { + "cpu": { + "$ref": "#/definitions/UsedResource" + }, + "from": { + "description": "From timestamp", + "type": "string", + "x-go-name": "From", + "example": "2006-01-02T15:04:05Z" + }, + "memory": { + "$ref": "#/definitions/UsedResource" + }, + "to": { + "description": "To timestamp", + "type": "string", + "x-go-name": "To", + "example": "2006-01-03T15:04:05Z" + }, + "warnings": { + "description": "Warning messages", + "type": "array", + "items": { + "type": "string" + }, + "x-go-name": "Warnings" + } + }, + "x-go-package": "github.com/equinor/radix-api/api/applications/models" + }, "X509Certificate": { "description": "X509Certificate holds information about a X509 certificate", "type": "object", From 4ce43bb2290e73ae991f193f09493ae8bb45f1e7 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Tue, 1 Oct 2024 15:43:52 +0200 Subject: [PATCH 2/3] Fixed query argument (#683) --- api/metrics/prometheus_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/metrics/prometheus_client.go b/api/metrics/prometheus_client.go index 718a15e5..24999e85 100644 --- a/api/metrics/prometheus_client.go +++ b/api/metrics/prometheus_client.go @@ -62,7 +62,7 @@ func getPrometheusQueries(appName, envName, componentName, duration, since strin environmentFilter := radixutils.TernaryString(envName == "", fmt.Sprintf(`,namespace=~"%s-.*"`, appName), fmt.Sprintf(`,namespace="%s"`, utils.GetEnvironmentNamespace(appName, envName))) - componentFilter := radixutils.TernaryString(envName == "", "", fmt.Sprintf(`,container="%s"`, componentName)) + componentFilter := radixutils.TernaryString(componentName == "", "", fmt.Sprintf(`,container="%s"`, componentName)) offsetFilter := radixutils.TernaryString(since == "", "", fmt.Sprintf(` offset %s `, since)) cpuUsageQuery := fmt.Sprintf(`sum by (namespace, container) (rate(container_cpu_usage_seconds_total{container!="", namespace!="%s-app" %s %s} [1h])) [%s:] %s`, appName, environmentFilter, componentFilter, duration, offsetFilter) memoryUsageQuery := fmt.Sprintf(`sum by (namespace, container) (container_memory_usage_bytes{container!="", namespace!="%s-app" %s %s} > 0) [%s:] %s`, appName, environmentFilter, componentFilter, duration, offsetFilter) From 02f2e2d8adadf43a5df08c18070d548ad5f6b7bb Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Wed, 2 Oct 2024 14:34:35 +0200 Subject: [PATCH 3/3] Rounded cpu and memory resources (#685) --- api/metrics/prometheus_handler.go | 5 +++-- api/metrics/prometheus_handler_test.go | 12 ++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/api/metrics/prometheus_handler.go b/api/metrics/prometheus_handler.go index 09bcd992..f09650cd 100644 --- a/api/metrics/prometheus_handler.go +++ b/api/metrics/prometheus_handler.go @@ -2,6 +2,7 @@ package metrics import ( "context" + "math" "regexp" "time" @@ -97,14 +98,14 @@ func parseQueryDuration(duration string, defaultValue string) (time.Duration, st func getCpuMetricValue(ctx context.Context, queryResults map[internal.QueryName]prometheusModel.Value, queryName internal.QueryName) *float64 { if value, ok := getMetricsValue(ctx, queryResults, queryName); ok { - return pointers.Ptr(value) + return pointers.Ptr(math.Round(value*1e6) / 1e6) } return nil } func getMemoryMetricValue(ctx context.Context, queryResults map[internal.QueryName]prometheusModel.Value, queryName internal.QueryName) *float64 { if value, ok := getMetricsValue(ctx, queryResults, queryName); ok { - return pointers.Ptr(value) + return pointers.Ptr(math.Round(value)) } return nil } diff --git a/api/metrics/prometheus_handler_test.go b/api/metrics/prometheus_handler_test.go index beb8abdd..8f2ff189 100644 --- a/api/metrics/prometheus_handler_test.go +++ b/api/metrics/prometheus_handler_test.go @@ -160,14 +160,14 @@ func getExpectedUsedResources(warnings ...string) *applicationModels.UsedResourc return &applicationModels.UsedResources{ Warnings: warnings, CPU: &applicationModels.UsedResource{ - Min: pointers.Ptr(0.02520196), - Avg: pointers.Ptr(0.0498679316), - Max: pointers.Ptr(0.134699898), + Min: pointers.Ptr(0.025202), + Avg: pointers.Ptr(0.049868), + Max: pointers.Ptr(0.1347), }, Memory: &applicationModels.UsedResource{ - Min: pointers.Ptr(168965.6892544), - Avg: pointers.Ptr(241308.93398770998), - Max: pointers.Ptr(358023.69331542), + Min: pointers.Ptr(168966.0), + Avg: pointers.Ptr(241309.0), + Max: pointers.Ptr(358024.0), }, } }