diff --git a/.dockerignore b/.dockerignore index 11c5a31..5425e9f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,3 @@ /azure-devops-exporter /release-assets +.env \ No newline at end of file diff --git a/README.md b/README.md index 08754e9..9e7aa58 100644 --- a/README.md +++ b/README.md @@ -42,17 +42,15 @@ Application Options: --azuredevops.agentpool= Enable scrape metrics for agent pool (IDs) [$AZURE_DEVOPS_AGENTPOOL] --whitelist.project= Filter projects (UUIDs) [$AZURE_DEVOPS_FILTER_PROJECT] --blacklist.project= Filter projects (UUIDs) [$AZURE_DEVOPS_BLACKLIST_PROJECT] + --timeline.state= Filter timeline states (completed, inProgress, pending) (default: completed) [$AZURE_DEVOPS_FILTER_TIMELINE_STATE] + --builds.all.project= Fetch all builds (even if they are not finished) [$AZURE_DEVOPS_FETCH_ALL_BUILDS_FILTER_PROJECT] --list.query= Pairs of query and project UUIDs in the form: '@' [$AZURE_DEVOPS_QUERIES] - --tags.schema= Tags to be extracted from builds in the format 'tagName:type' with following types: number, - info, bool [$AZURE_DEVOPS_TAG_SCHEMA] + --tags.schema= Tags to be extracted from builds in the format 'tagName:type' with following types: number, info, bool [$AZURE_DEVOPS_TAG_SCHEMA] --tags.build.definition= Build definition ids to query tags (IDs) [$AZURE_DEVOPS_TAG_BUILD_DEFINITION] - --cache.path= Cache path (to folder, file://path... or - azblob://storageaccount.blob.core.windows.net/containername or - k8scm://{namespace}/{configmap}}) [$CACHE_PATH] + --cache.path= Cache path (to folder, file://path... or azblob://storageaccount.blob.core.windows.net/containername or k8scm://{namespace}/{configmap}}) [$CACHE_PATH] --request.concurrency= Number of concurrent requests against dev.azure.com (default: 10) [$REQUEST_CONCURRENCY] --request.retries= Number of retried requests against dev.azure.com (default: 3) [$REQUEST_RETRIES] - --servicediscovery.refresh= Refresh duration for servicediscovery (time.duration) (default: 30m) - [$SERVICEDISCOVERY_REFRESH] + --servicediscovery.refresh= Refresh duration for servicediscovery (time.duration) (default: 30m) [$SERVICEDISCOVERY_REFRESH] --limit.project= Limit number of projects (default: 100) [$LIMIT_PROJECT] --limit.builds-per-project= Limit builds per project (default: 100) [$LIMIT_BUILDS_PER_PROJECT] --limit.builds-per-definition= Limit builds per definition (default: 10) [$LIMIT_BUILDS_PER_DEFINITION] @@ -60,10 +58,8 @@ Application Options: --limit.releases-per-definition= Limit releases per definition (default: 100) [$LIMIT_RELEASES_PER_DEFINITION] --limit.deployments-per-definition= Limit deployments per definition (default: 100) [$LIMIT_DEPLOYMENTS_PER_DEFINITION] --limit.releasedefinitions-per-project= Limit builds per definition (default: 100) [$LIMIT_RELEASEDEFINITION_PER_PROJECT] - --limit.build-history-duration= Time (time.Duration) how long the exporter should look back for builds (default: 48h) - [$LIMIT_BUILD_HISTORY_DURATION] - --limit.release-history-duration= Time (time.Duration) how long the exporter should look back for releases (default: 48h) - [$LIMIT_RELEASE_HISTORY_DURATION] + --limit.build-history-duration= Time (time.Duration) how long the exporter should look back for builds (default: 48h) [$LIMIT_BUILD_HISTORY_DURATION] + --limit.release-history-duration= Time (time.Duration) how long the exporter should look back for releases (default: 48h) [$LIMIT_RELEASE_HISTORY_DURATION] --server.bind= Server address (default: :8080) [$SERVER_BIND] --server.timeout.read= Server read timeout (default: 5s) [$SERVER_TIMEOUT_READ] --server.timeout.write= Server write timeout (default: 10s) [$SERVER_TIMEOUT_WRITE] diff --git a/azure-devops-client/build.go b/azure-devops-client/build.go index bcb08d9..5880382 100644 --- a/azure-devops-client/build.go +++ b/azure-devops-client/build.go @@ -53,6 +53,7 @@ type TimelineRecord struct { Result string `json:"result"` WorkerName string `json:"workerName"` Identifier string `json:"identifier"` + State string `json:"state"` StartTime time.Time FinishTime time.Time } @@ -195,14 +196,26 @@ func (c *AzureDevopsClient) ListBuildHistoryWithStatus(project string, minTime t defer c.concurrencyUnlock() c.concurrencyLock() - url := fmt.Sprintf( - "%v/_apis/build/builds?api-version=%v&minTime=%s&statusFilter=%v", - url.QueryEscape(project), - url.QueryEscape(c.ApiVersion), - url.QueryEscape(minTime.Format(time.RFC3339)), - url.QueryEscape(statusFilter), - ) - response, err := c.rest().R().Get(url) + requestUrl := "" + + if statusFilter == "all" { + requestUrl = fmt.Sprintf( + "%v/_apis/build/builds?api-version=%v&statusFilter=%v", + url.QueryEscape(project), + url.QueryEscape(c.ApiVersion), + url.QueryEscape(statusFilter), + ) + } else { + requestUrl = fmt.Sprintf( + "%v/_apis/build/builds?api-version=%v&minTime=%s&statusFilter=%v", + url.QueryEscape(project), + url.QueryEscape(c.ApiVersion), + url.QueryEscape(minTime.Format(time.RFC3339)), + url.QueryEscape(statusFilter), + ) + } + + response, err := c.rest().R().Get(requestUrl) if err := c.checkResponse(response, err); err != nil { error = err return @@ -214,6 +227,18 @@ func (c *AzureDevopsClient) ListBuildHistoryWithStatus(project string, minTime t return } + // if the status filter is "all", we need to filter the builds by minTime manually because Azure DevOps API does not support it + if statusFilter == "all" { + var filteredList BuildList + for _, build := range list.List { + if build.StartTime.After(minTime) { + filteredList.List = append(filteredList.List, build) + } + } + filteredList.Count = len(filteredList.List) + list = filteredList + } + return } diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..cbf422c --- /dev/null +++ b/compose.yaml @@ -0,0 +1,16 @@ +services: + exporter: + build: . + ports: + - "8000:8000" + environment: + SERVER_BIND: ${SERVER_BIND} + AZURE_DEVOPS_URL: ${AZURE_DEVOPS_URL} + AZURE_DEVOPS_ORGANISATION: ${AZURE_DEVOPS_ORGANISATION} + AZURE_DEVOPS_ACCESS_TOKEN: ${AZURE_DEVOPS_ACCESS_TOKEN} + AZURE_DEVOPS_FILTER_PROJECT: 72690669-de93-4a98-84a9-8300ce32a2f2 + LIMIT_BUILDS_PER_PROJECT: 500 + LIMIT_BUILDS_PER_DEFINITION: 100 + SERVER_TIMEOUT_READ: 15s + AZURE_DEVOPS_FETCH_ALL_BUILDS: "true" + #AZURE_DEVOPS_FILTER_TIMELINE_STATE: "completed inProgress pending" \ No newline at end of file diff --git a/config/opts.go b/config/opts.go index 79b5dc6..bd9ae5f 100644 --- a/config/opts.go +++ b/config/opts.go @@ -58,6 +58,9 @@ type ( FilterProjects []string `long:"whitelist.project" env:"AZURE_DEVOPS_FILTER_PROJECT" env-delim:" " description:"Filter projects (UUIDs)"` BlacklistProjects []string `long:"blacklist.project" env:"AZURE_DEVOPS_BLACKLIST_PROJECT" env-delim:" " description:"Filter projects (UUIDs)"` + FilterTimelineState []string `long:"timeline.state" env:"AZURE_DEVOPS_FILTER_TIMELINE_STATE" env-delim:" " description:"Filter timeline states (completed, inProgress, pending)" default:"completed"` + FetchAllBuildsFilter []string `long:"builds.all.project" env:"AZURE_DEVOPS_FETCH_ALL_BUILDS_FILTER_PROJECT" env-delim:" " description:"Fetch all builds from projects (UUIDs or names)"` + // query settings QueriesWithProjects []string `long:"list.query" env:"AZURE_DEVOPS_QUERIES" env-delim:" " description:"Pairs of query and project UUIDs in the form: '@'"` diff --git a/metrics_build.go b/metrics_build.go index 52283e9..264d693 100644 --- a/metrics_build.go +++ b/metrics_build.go @@ -303,8 +303,20 @@ func (m *MetricsCollectorBuild) collectBuilds(ctx context.Context, logger *zap.S } func (m *MetricsCollectorBuild) collectBuildsTimeline(ctx context.Context, logger *zap.SugaredLogger, callback chan<- func(), project devopsClient.Project) { +<<<<<<< main + minTime := time.Now().Add(-opts.Limit.BuildHistoryDuration) + + statusFilter := "completed" + if arrayStringContains(opts.AzureDevops.FetchAllBuildsFilter, project.Name) || arrayStringContains(opts.AzureDevops.FetchAllBuildsFilter, project.Id) { + logger.Info("fetching all builds for project " + project.Name) + statusFilter = "all" + } + + list, err := AzureDevopsClient.ListBuildHistoryWithStatus(project.Id, minTime, statusFilter) +======= minTime := time.Now().Add(-Opts.Limit.BuildHistoryDuration) list, err := AzureDevopsClient.ListBuildHistoryWithStatus(project.Id, minTime, "completed") +>>>>>>> main if err != nil { logger.Error(err) return @@ -316,8 +328,18 @@ func (m *MetricsCollectorBuild) collectBuildsTimeline(ctx context.Context, logge buildTaskMetric := m.Collector.GetMetricList("buildTask") for _, build := range list.List { + timelineRecordList, _ := AzureDevopsClient.ListBuildTimeline(project.Id, int64ToString(build.Id)) for _, timelineRecord := range timelineRecordList.List { + + if opts.AzureDevops.FilterTimelineState != nil && !arrayStringContains(opts.AzureDevops.FilterTimelineState, timelineRecord.State) { + continue + } + + if timelineRecord.Result == "" { + timelineRecord.Result = "unknown" + } + recordType := timelineRecord.RecordType switch strings.ToLower(recordType) { case "stage": @@ -635,8 +657,20 @@ func (m *MetricsCollectorBuild) collectBuildsTimeline(ctx context.Context, logge } func (m *MetricsCollectorBuild) collectBuildsTags(ctx context.Context, logger *zap.SugaredLogger, callback chan<- func(), project devopsClient.Project) { +<<<<<<< main + minTime := time.Now().Add(-opts.Limit.BuildHistoryDuration) + + statusFilter := "completed" + if arrayStringContains(opts.AzureDevops.FetchAllBuildsFilter, project.Name) || arrayStringContains(opts.AzureDevops.FetchAllBuildsFilter, project.Id) { + logger.Info("fetching all builds for project " + project.Name) + statusFilter = "all" + } + + list, err := AzureDevopsClient.ListBuildHistoryWithStatus(project.Id, minTime, statusFilter) +======= minTime := time.Now().Add(-Opts.Limit.BuildHistoryDuration) list, err := AzureDevopsClient.ListBuildHistoryWithStatus(project.Id, minTime, "completed") +>>>>>>> main if err != nil { logger.Error(err) return