From 6a560356cdeb0befde19147cbe1bcb75d88c407b Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 12 Jul 2023 17:08:30 +0200 Subject: [PATCH] Use artifats-snapshot endpoint when artifacts-api can't find an artefact (#3574) * Use artifats-snapshot endpoint instead of artifacts-api * fixup * fixup * fixup * fixup * use json.parse * Added snapshots resolver * Fixed version picking * Added a test --- .../dra/snapshot_artifacts_test.json | 50 ++++ pkg/downloads/releases.go | 258 ++++++++++++++++++ pkg/downloads/releases_test.go | 130 +++++++++ pkg/downloads/versions.go | 12 + 4 files changed, 450 insertions(+) create mode 100644 pkg/_testresources/dra/snapshot_artifacts_test.json create mode 100644 pkg/downloads/releases_test.go diff --git a/pkg/_testresources/dra/snapshot_artifacts_test.json b/pkg/_testresources/dra/snapshot_artifacts_test.json new file mode 100644 index 0000000000..41fedb5919 --- /dev/null +++ b/pkg/_testresources/dra/snapshot_artifacts_test.json @@ -0,0 +1,50 @@ +{ + "branch": "8.9", + "release_branch": "8.9", + "version": "8.9.0-SNAPSHOT", + "build_id": "8.9.0-b6405422", + "start_time": "Tue, 11 Jul 2023 19:39:23 GMT", + "end_time": "Tue, 11 Jul 2023 19:45:28 GMT", + "build_duration_seconds": 365, + "manifest_version": "2.1.0", + "prefix": "beats", + "projects": { + "beats": { + "branch": "8.9", + "commit_hash": "ae84feba9e48c163a3cb681b777607cab02b959d", + "commit_url": "https://github.com/elastic/beats/commits/ae84feba9e48c163a3cb681b777607cab02b959d", + "build_duration_seconds": 0, + "packages": { + "auditbeat-8.9.0-SNAPSHOT-windows-x86_64.zip": { + "url": "https://artifacts-snapshot.elastic.co/beats/8.9.0-b6405422/downloads/beats/auditbeat/auditbeat-8.9.0-SNAPSHOT-windows-x86_64.zip", + "sha_url": "https://artifacts-snapshot.elastic.co/beats/8.9.0-b6405422/downloads/beats/auditbeat/auditbeat-8.9.0-SNAPSHOT-windows-x86_64.zip.sha512", + "type": "zip", + "architecture": "x86_64", + "os": [ + "windows" + ] + }, + "auditbeat-8.9.0-SNAPSHOT-amd64.deb": { + "url": "https://artifacts-snapshot.elastic.co/auditbeat-8.9.0-SNAPSHOT-amd64.deb", + "sha_url": "https://artifacts-snapshot.elastic.co/auditbeat-8.9.0-SNAPSHOT-amd64.deb.sha512", + "type": "deb", + "architecture": "amd64", + "attributes": { + "include_in_repo": "true", + "oss": "false" + } + }, + "auditbeat-8.9.0-SNAPSHOT-x86_64.rpm": { + "url": "https://artifacts-snapshot.elastic.co/beats/8.9.0-b6405422/downloads/beats/auditbeat/auditbeat-8.9.0-SNAPSHOT-x86_64.rpm", + "sha_url": "https://artifacts-snapshot.elastic.co/beats/8.9.0-b6405422/downloads/beats/auditbeat/auditbeat-8.9.0-SNAPSHOT-x86_64.rpm.sha512", + "type": "rpm", + "architecture": "x86_64", + "attributes": { + "include_in_repo": "true", + "oss": "false" + } + } + } + } + } +} \ No newline at end of file diff --git a/pkg/downloads/releases.go b/pkg/downloads/releases.go index 37c4078ca9..14faebbf22 100644 --- a/pkg/downloads/releases.go +++ b/pkg/downloads/releases.go @@ -5,6 +5,7 @@ package downloads import ( + "encoding/json" "fmt" "strings" "time" @@ -153,6 +154,263 @@ func (r *ArtifactURLResolver) Resolve() (string, string, error) { return downloadURL, downloadshaURL, nil } +type ArtifactsSnapshotVersion struct { + Host string +} + +func newArtifactsSnapshotCustom(host string) *ArtifactsSnapshotVersion { + return &ArtifactsSnapshotVersion{ + Host: host, + } +} + +// Uses artifacts-snapshot.elastic.co to retrieve the latest version of a SNAPSHOT artifact +func NewArtifactsSnapshot() *ArtifactsSnapshotVersion { + return &ArtifactsSnapshotVersion{ + Host: "https://artifacts-snapshot.elastic.co", + } +} + +// GetSnapshotArtifactVersion returns the current version: +// Uses artifacts-snapshot.elastic.co to retrieve the latest version of a SNAPSHOT artifact +// 1. Elastic's artifact repository, building the JSON path query based +// If the version is a SNAPSHOT including a commit, then it will directly use the version without checking the artifacts API +// i.e. GetSnapshotArtifactVersion("$VERSION-abcdef-SNAPSHOT") +func (as *ArtifactsSnapshotVersion) GetSnapshotArtifactVersion(version string) (string, error) { + cacheKey := fmt.Sprintf("%s/beats/latest/%s.json", as.Host, version) + + if val, ok := elasticVersionsCache[cacheKey]; ok { + log.WithFields(log.Fields{ + "URL": cacheKey, + "version": val, + }).Debug("Retrieving version from local cache") + return val, nil + } + + if SnapshotHasCommit(version) { + elasticVersionsCache[cacheKey] = version + return version, nil + } + + exp := utils.GetExponentialBackOff(time.Minute) + + retryCount := 1 + + body := "" + + apiStatus := func() error { + r := curl.HTTPRequest{ + URL: cacheKey, + } + + response, err := curl.Get(r) + if err != nil { + log.WithFields(log.Fields{ + "version": version, + "error": err, + "retry": retryCount, + "statusEndpoint": r.URL, + "elapsedTime": exp.GetElapsedTime(), + }).Warn("The Elastic artifacts API is not available yet") + + retryCount++ + + return err + } + + log.WithFields(log.Fields{ + "retries": retryCount, + "statusEndpoint": r.URL, + "elapsedTime": exp.GetElapsedTime(), + }).Debug("The Elastic artifacts API is available") + + body = response + return nil + } + + err := backoff.Retry(apiStatus, exp) + if err != nil { + return "", err + } + + type ArtifactsSnapshotResponse struct { + Version string `json:"version"` // example value: "8.8.3-SNAPSHOT" + BuildID string `json:"build_id"` // example value: "8.8.3-b1d8691a" + ManifestURL string `json:"manifest_url"` // example value: https://artifacts-snapshot.elastic.co/beats/8.8.3-b1d8691a/manifest-8.8.3-SNAPSHOT.json + SummaryURL string `json:"summary_url"` // example value: https://artifacts-snapshot.elastic.co/beats/8.8.3-b1d8691a/summary-8.8.3-SNAPSHOT.html + } + response := ArtifactsSnapshotResponse{} + err = json.Unmarshal([]byte(body), &response) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "version": version, + "body": body, + }).Error("Could not parse the response body to retrieve the version") + + return "", fmt.Errorf("could not parse the response body to retrieve the version: %w", err) + } + + hashParts := strings.Split(response.BuildID, "-") + if (len(hashParts) < 2) || (hashParts[1] == "") { + log.WithFields(log.Fields{ + "buildId": response.BuildID, + }).Error("Could not parse the build_id to retrieve the version hash") + return "", fmt.Errorf("could not parse the build_id to retrieve the version hash: %s", response.BuildID) + } + hash := hashParts[1] + parsedVersion := hashParts[0] + + latestVersion := fmt.Sprintf("%s-%s-SNAPSHOT", parsedVersion, hash) + + log.WithFields(log.Fields{ + "alias": version, + "version": latestVersion, + }).Debug("Latest version for current version obtained") + + elasticVersionsCache[cacheKey] = latestVersion + + return latestVersion, nil +} + +// NewArtifactURLResolver creates a new resolver for artifacts that are currently in development, from the artifacts API +func NewArtifactSnapshotURLResolver(fullName string, name string, version string) DownloadURLResolver { + return newCustomSnapshotURLResolver(fullName, name, version, "https://artifacts-snapshot.elastic.co") +} + +// For testing purposes +func newCustomSnapshotURLResolver(fullName string, name string, version string, host string) DownloadURLResolver { + // resolve version alias + resolvedVersion, err := newArtifactsSnapshotCustom(host).GetSnapshotArtifactVersion(version) + if err != nil { + return nil + } + return &ArtifactsSnapshotURLResolver{ + FullName: fullName, + Name: name, + Version: resolvedVersion, + SnapshotApiHost: host, + } +} + +// ArtifactsSnapshotURLResolver type to resolve the URL of artifacts that are currently in development, from the artifacts API +type ArtifactsSnapshotURLResolver struct { + FullName string + Name string + Version string + SnapshotApiHost string +} + +func (asur *ArtifactsSnapshotURLResolver) Resolve() (string, string, error) { + artifactName := asur.FullName + artifact := asur.Name + version := asur.Version + commit, err := ExtractCommitHash(version) + semVer := GetVersion(version) + if err != nil { + log.WithFields(log.Fields{ + "artifact": artifact, + "artifactName": artifactName, + "version": version, + }).Info("The version does not contain a commit hash, it is not a snapshot") + return "", "", err + } + + exp := utils.GetExponentialBackOff(time.Minute) + + retryCount := 1 + + body := "" + + apiStatus := func() error { + r := curl.HTTPRequest{ + // https://artifacts-snapshot.elastic.co/beats/8.9.0-d1b14479/manifest-8.9.0-SNAPSHOT.json + URL: fmt.Sprintf("%s/beats/%s-%s/manifest-%s-SNAPSHOT.json", asur.SnapshotApiHost, semVer, commit, semVer), + } + + response, err := curl.Get(r) + if err != nil { + log.WithFields(log.Fields{ + "artifact": artifact, + "artifactName": artifactName, + "version": version, + "error": err, + "retry": retryCount, + "statusEndpoint": r.URL, + "elapsedTime": exp.GetElapsedTime(), + }).Warn("The Elastic artifacts API is not available yet") + + retryCount++ + + return err + } + + log.WithFields(log.Fields{ + "retries": retryCount, + "statusEndpoint": r.URL, + "elapsedTime": exp.GetElapsedTime(), + }).Debug("The Elastic artifacts API is available") + + body = response + return nil + } + + err = backoff.Retry(apiStatus, exp) + if err != nil { + return "", "", err + } + + var jsonParsed map[string]interface{} + err = json.Unmarshal([]byte(body), &jsonParsed) + if err != nil { + log.WithFields(log.Fields{ + "artifact": artifact, + "artifactName": artifactName, + "version": version, + }).Error("Could not parse the response body for the artifact") + return "", "", err + } + + url, shaURL, err := findSnapshotPackage(jsonParsed, artifactName) + if err != nil { + return "", "", err + } + + log.WithFields(log.Fields{ + "retries": retryCount, + "artifact": artifact, + "artifactName": artifactName, + "elapsedTime": exp.GetElapsedTime(), + "version": version, + }).Trace("Artifact found") + + return url, shaURL, nil +} + +func findSnapshotPackage(jsonParsed map[string]interface{}, fullName string) (string, string, error) { + projects, ok := jsonParsed["projects"].(map[string]interface{}) + if !ok { + return "", "", fmt.Errorf("key 'projects' does not exist") + } + + for _, project := range projects { + projectPackages, ok := project.(map[string]interface{})["packages"].(map[string]interface{}) + if !ok { + continue + } + + pack, ok := projectPackages[fullName].(map[string]interface{}) + + if !ok { + continue + } + + return pack["url"].(string), pack["sha_url"].(string), nil + + } + return "", "", fmt.Errorf("package %s not found", fullName) +} + // ReleaseURLResolver type to resolve the URL of downloads that are currently published in elastic.co/downloads type ReleaseURLResolver struct { Project string diff --git a/pkg/downloads/releases_test.go b/pkg/downloads/releases_test.go new file mode 100644 index 0000000000..17e0abc963 --- /dev/null +++ b/pkg/downloads/releases_test.go @@ -0,0 +1,130 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package downloads + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetSnapshotArtifactVersion(t *testing.T) { + t.Run("Positive: parses commit has and returns full version", func(t *testing.T) { + mockResponse := `{ + "version" : "8.8.3-SNAPSHOT", + "build_id" : "8.8.3-b1d8691a", + "manifest_url" : "https://artifacts-snapshot.elastic.co/beats/8.8.3-b1d8691a/manifest-8.8.3-SNAPSHOT.json", + "summary_url" : "https://artifacts-snapshot.elastic.co/beats/8.8.3-b1d8691a/summary-8.8.3-SNAPSHOT.html" + }` + + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprintln(w, mockResponse) + })) + defer mockServer.Close() + + mockURL := mockServer.URL + "/beats/latest/8.8.3-SNAPSHOT.json" + artifactsSnapshot := newArtifactsSnapshotCustom(mockURL) + version, err := artifactsSnapshot.GetSnapshotArtifactVersion("8.8.3-SNAPSHOT") + assert.NoError(t, err, "Expected no error") + assert.Equal(t, "8.8.3-b1d8691a-SNAPSHOT", version, "Expected version to match") + }) + + t.Run("Negative: Invalid json response from server", func(t *testing.T) { + mockResponse := `sdf{ + "ver" : "8.8.3-SNAPSHOT", + "bui" : "8.8.3-b1d8691a", + "manifest_url" : "https://artifacts-snapshot.elastic.co/beats/8.8.3-b1d8691a/manifest-8.8.3-SNAPSHOT.json", + "summary_url" : "https://artifacts-snapshot.elastic.co/beats/8.8.3-b1d8691a/summary-8.8.3-SNAPSHOT.html" + }` + + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprintln(w, mockResponse) + })) + defer mockServer.Close() + + mockURL := mockServer.URL + "/beats/latest/8.8.3-SNAPSHOT.json" + artifactsSnapshot := newArtifactsSnapshotCustom(mockURL) + version, err := artifactsSnapshot.GetSnapshotArtifactVersion("8.8.3-SNAPSHOT") + assert.ErrorContains(t, err, "could not parse the response body") + assert.Empty(t, version) + }) + + t.Run("Negative: Unexpected build_id format", func(t *testing.T) { + mockResponse := `{ + "version" : "8.8.3-SNAPSHOT", + "build_id" : "bd8691a", + "manifest_url" : "https://artifacts-snapshot.elastic.co/beats/8.8.3-b1d8691a/manifest-8.8.3-SNAPSHOT.json", + "summary_url" : "https://artifacts-snapshot.elastic.co/beats/8.8.3-b1d8691a/summary-8.8.3-SNAPSHOT.html" + }` + + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprintln(w, mockResponse) + })) + defer mockServer.Close() + + mockURL := mockServer.URL + "/beats/latest/8.8.3-SNAPSHOT.json" + artifactsSnapshot := newArtifactsSnapshotCustom(mockURL) + version, err := artifactsSnapshot.GetSnapshotArtifactVersion("8.8.3-SNAPSHOT") + assert.ErrorContains(t, err, "could not parse the build_id") + assert.ErrorContains(t, err, "bd8691a") + assert.Empty(t, version) + }) +} + +type MockHandler struct { + Responses map[string]string +} + +func (m *MockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + path := r.URL.Path + + response, ok := m.Responses[path] + if !ok { + http.NotFound(w, r) + return + } + + fmt.Fprint(w, response) +} + +func TestArtifactsSnapshotResolver(t *testing.T) { + + t.Run("Positive: parses commit has and returns full version", func(t *testing.T) { + + artifactsSnapshotMockManifest, err := os.ReadFile(filepath.Join("..", "_testresources", "dra", "snapshot_artifacts_test.json")) + require.NoError(t, err, "couldn't read pkg/_testresources/dra/snapshot_artifacts_test.json") + mockResponses := map[string]string{ + "/beats/latest/8.9.0-SNAPSHOT.json": `{ + "version" : "8.9.0-SNAPSHOT", + "build_id" : "8.9.0-b6405422", + "manifest_url" : "https://artifacts-snapshot.elastic.co/beats/8.9.0-b6405422/manifest-8.9.0-SNAPSHOT.json", + "summary_url" : "https://artifacts-snapshot.elastic.co/beats/8.9.0-b6405422/summary-8.9.0-SNAPSHOT.html" + }`, + "/beats/8.9.0-b6405422/manifest-8.9.0-SNAPSHOT.json": string(artifactsSnapshotMockManifest), + } + + mockHandler := &MockHandler{ + Responses: mockResponses, + } + + server := httptest.NewServer(mockHandler) + defer server.Close() + + urlResolver := newCustomSnapshotURLResolver("auditbeat-8.9.0-SNAPSHOT-amd64.deb", "auditbeat", "8.9.0-SNAPSHOT", server.URL) + url, shaUrl, err := urlResolver.Resolve() + assert.Equal(t, "https://artifacts-snapshot.elastic.co/auditbeat-8.9.0-SNAPSHOT-amd64.deb", url) + assert.Equal(t, "https://artifacts-snapshot.elastic.co/auditbeat-8.9.0-SNAPSHOT-amd64.deb.sha512", shaUrl) + assert.NoError(t, err, "Expected no error") + }) +} diff --git a/pkg/downloads/versions.go b/pkg/downloads/versions.go index 66ac55f862..88fe3388d5 100644 --- a/pkg/downloads/versions.go +++ b/pkg/downloads/versions.go @@ -266,6 +266,17 @@ func RemoveCommitFromSnapshot(s string) string { return re.ReplaceAllString(s, "") } +func ExtractCommitHash(input string) (string, error) { + re := regexp.MustCompile(`-(\w+)-`) + matches := re.FindStringSubmatch(input) + + if len(matches) < 2 { + return "", fmt.Errorf("commit hash not found") + } + + return matches[1], nil +} + // SnapshotHasCommit returns true if the snapshot version contains a commit format func SnapshotHasCommit(s string) bool { // regex = X.Y.Z-commit-SNAPSHOT @@ -469,6 +480,7 @@ func FetchProjectBinaryForSnapshots(ctx context.Context, useCISnapshots bool, pr downloadURLResolvers := []DownloadURLResolver{ NewReleaseURLResolver(elasticAgentNamespace, artifactName, artifact), NewArtifactURLResolver(artifactName, artifact, version), + NewArtifactSnapshotURLResolver(artifactName, artifact, version), } downloadURL, downloadShaURL, err = getDownloadURLFromResolvers(downloadURLResolvers) if err != nil {