Skip to content

Commit

Permalink
feat: skip experimental versions on scrape
Browse files Browse the repository at this point in the history
  • Loading branch information
@RaduPetreTarean
RaduPetreTarean committed Jul 17, 2024
1 parent 1073ffe commit 3b1628b
Show file tree
Hide file tree
Showing 4 changed files with 71 additions and 32 deletions.
32 changes: 22 additions & 10 deletions internal/scraper/gcs_scraper_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ func TestGCSScraper(t *testing.T) {
tests := []struct {
service, version, digest string
}{
{"petfood", "2021-09-01", "sha256:zCgJaPeR8R21wsAlYn46xO6NE3XJiyFtLnYrP4DpM3U="},
{"petfood", "2021-09-01~experimental", "sha256:zCgJaPeR8R21wsAlYn46xO6NE3XJiyFtLnYrP4DpM3U="},
{"animals", "2021-10-16", "sha256:hcv2i7awT6CcSCecw9WrYBokFyzYNVaQArGgqHqdj7s="},
}

Expand Down Expand Up @@ -61,14 +61,20 @@ func TestGCSScraper(t *testing.T) {

// Version digests now known to storage
for _, test := range tests {
ok, err := st.HasVersion(ctx, test.service, test.version, test.digest)
c.Assert(err, qt.IsNil)
c.Assert(ok, qt.IsTrue)
if scraper.IsExperimentalVersion(test.version) {
ok, err := st.HasVersion(ctx, test.service, test.version, test.digest)
c.Assert(err, qt.IsNil)
c.Assert(ok, qt.IsFalse, qt.Commentf("experimental version %s should not be included", test.version))
} else {
ok, err := st.HasVersion(ctx, test.service, test.version, test.digest)
c.Assert(err, qt.IsNil)
c.Assert(ok, qt.IsTrue)
}
}

vi, err := st.VersionIndex(ctx)
c.Assert(err, qt.IsNil)
c.Assert(len(vi.Versions()), qt.Equals, 4)
c.Assert(len(vi.Versions()), qt.Equals, 3)
for _, version := range vi.Versions() {
specData, err := st.Version(ctx, version.String())
c.Assert(err, qt.IsNil)
Expand All @@ -95,7 +101,7 @@ func TestGCSScraperCollation(t *testing.T) {
tests := []struct {
service, version, digest string
}{
{"petfood", "2021-09-01", "sha256:zCgJaPeR8R21wsAlYn46xO6NE3XJiyFtLnYrP4DpM3U="},
{"petfood", "2021-09-01~experimental", "sha256:zCgJaPeR8R21wsAlYn46xO6NE3XJiyFtLnYrP4DpM3U="},
{"animals", "2021-10-16", "sha256:hcv2i7awT6CcSCecw9WrYBokFyzYNVaQArGgqHqdj7s="},
}

Expand Down Expand Up @@ -125,14 +131,20 @@ func TestGCSScraperCollation(t *testing.T) {

// Version digests now known to storage
for _, test := range tests {
ok, err := st.HasVersion(ctx, test.service, test.version, test.digest)
c.Assert(err, qt.IsNil)
c.Assert(ok, qt.IsTrue)
if scraper.IsExperimentalVersion(test.version) {
ok, err := st.HasVersion(ctx, test.service, test.version, test.digest)
c.Assert(err, qt.IsNil)
c.Assert(ok, qt.IsFalse, qt.Commentf("experimental version %s should not be included", test.version))
} else {
ok, err := st.HasVersion(ctx, test.service, test.version, test.digest)
c.Assert(err, qt.IsNil)
c.Assert(ok, qt.IsTrue)
}
}

vi, err := st.VersionIndex(ctx)
c.Assert(err, qt.IsNil)
c.Assert(len(vi.Versions()), qt.Equals, 4)
c.Assert(len(vi.Versions()), qt.Equals, 3)
for _, version := range vi.Versions() {
specData, err := st.Version(ctx, version.String())
c.Assert(err, qt.IsNil)
Expand Down
32 changes: 22 additions & 10 deletions internal/scraper/s3_scraper_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ func TestS3Scraper(t *testing.T) {
tests := []struct {
name, version, digest string
}{
{"petfood", "2021-09-01", "sha256:zCgJaPeR8R21wsAlYn46xO6NE3XJiyFtLnYrP4DpM3U="},
{"petfood", "2021-09-01~experimental", "sha256:zCgJaPeR8R21wsAlYn46xO6NE3XJiyFtLnYrP4DpM3U="},
{"animals", "2021-10-16", "sha256:hcv2i7awT6CcSCecw9WrYBokFyzYNVaQArGgqHqdj7s="},
}

Expand Down Expand Up @@ -57,14 +57,20 @@ func TestS3Scraper(t *testing.T) {

// Version digests now known to storage
for _, test := range tests {
ok, err := st.HasVersion(ctx, test.name, test.version, test.digest)
c.Assert(err, qt.IsNil)
c.Assert(ok, qt.IsTrue)
if scraper.IsExperimentalVersion(test.version) {
ok, err := st.HasVersion(ctx, test.name, test.version, test.digest)
c.Assert(err, qt.IsNil)
c.Assert(ok, qt.IsFalse, qt.Commentf("experimental version %s should not be included", test.version))
} else {
ok, err := st.HasVersion(ctx, test.name, test.version, test.digest)
c.Assert(err, qt.IsNil)
c.Assert(ok, qt.IsTrue)
}
}

vi, err := st.VersionIndex(ctx)
c.Assert(err, qt.IsNil)
c.Assert(len(vi.Versions()), qt.Equals, 4)
c.Assert(len(vi.Versions()), qt.Equals, 3)
for _, version := range vi.Versions() {
specData, err := st.Version(ctx, version.String())
c.Assert(err, qt.IsNil)
Expand All @@ -90,7 +96,7 @@ func TestS3ScraperCollation(t *testing.T) {
tests := []struct {
name, version, digest string
}{{
"petfood", "2021-09-01", "sha256:zCgJaPeR8R21wsAlYn46xO6NE3XJiyFtLnYrP4DpM3U=",
"petfood", "2021-09-01~experimental", "sha256:zCgJaPeR8R21wsAlYn46xO6NE3XJiyFtLnYrP4DpM3U=",
}, {
"animals", "2021-10-16", "sha256:hcv2i7awT6CcSCecw9WrYBokFyzYNVaQArGgqHqdj7s=",
}}
Expand All @@ -117,14 +123,20 @@ func TestS3ScraperCollation(t *testing.T) {

// Version digests now known to storage
for _, test := range tests {
ok, err := st.HasVersion(ctx, test.name, test.version, test.digest)
c.Assert(err, qt.IsNil)
c.Assert(ok, qt.IsTrue)
if scraper.IsExperimentalVersion(test.version) {
ok, err := st.HasVersion(ctx, test.name, test.version, test.digest)
c.Assert(err, qt.IsNil)
c.Assert(ok, qt.IsFalse, qt.Commentf("experimental version %s should not be included", test.version))
} else {
ok, err := st.HasVersion(ctx, test.name, test.version, test.digest)
c.Assert(err, qt.IsNil)
c.Assert(ok, qt.IsTrue)
}
}

vi, err := st.VersionIndex(ctx)
c.Assert(err, qt.IsNil)
c.Assert(len(vi.Versions()), qt.Equals, 4)
c.Assert(len(vi.Versions()), qt.Equals, 3)
for _, version := range vi.Versions() {
specData, err := st.Version(ctx, version.String())
c.Assert(err, qt.IsNil)
Expand Down
9 changes: 9 additions & 0 deletions internal/scraper/scraper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,11 @@ func (s *Scraper) scrape(ctx context.Context, scrapeTime time.Time, svc service)
// TODO: we might run this concurrently per live service pod if/when
// we're more k8s aware, but we won't do that yet.

// Skip if it's an experimental version
if IsExperimentalVersion(versions[i]) {
continue
}

// Skip if it's a legacy api using the default legacy version.
if isLegacyVersion(versions[i]) {
continue
Expand Down Expand Up @@ -290,3 +295,7 @@ func isLegacyVersion(version string) bool {
// This default version predates vervet's creation date.
return version == "2021-01-01"
}

func IsExperimentalVersion(version string) bool {
return strings.HasSuffix(version, "~experimental")
}
30 changes: 18 additions & 12 deletions internal/scraper/scraper_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,17 @@ import (
var (
t0 = time.Date(2021, time.December, 3, 20, 49, 51, 0, time.UTC)
collatedPaths = map[string]int{
"2021-09-01": 1,
"2021-09-16": 2,
"2021-10-01": 3,
"2021-10-16": 4,
"2021-09-01~experimental": 1,
"2021-09-16": 2,
"2021-10-01": 3,
"2021-10-16": 4,
}

petfood = &testService{
versions: []string{"2021-09-01", "2021-09-16"},
versions: []string{"2021-09-01~experimental", "2021-09-16"},
contents: map[string]string{
"2021-09-01": `{"paths":{"/crickets": {"get": {}}}}`,
"2021-09-16": `{"paths":{"/crickets": {"get": {}}, "/kibble": {"get": {}}}}`,
"2021-09-01~experimental": `{"paths":{"/crickets": {"get": {}}}}`,
"2021-09-16": `{"paths":{"/crickets": {"get": {}}, "/kibble": {"get": {}}}}`,
},
}
animals = &testService{
Expand Down Expand Up @@ -85,7 +85,7 @@ func TestScraper(t *testing.T) {
tests := []struct {
name, version, digest string
}{
{"petfood", "2021-09-01", "sha256:zCgJaPeR8R21wsAlYn46xO6NE3XJiyFtLnYrP4DpM3U="},
{"petfood", "2021-09-01~experimental", "sha256:zCgJaPeR8R21wsAlYn46xO6NE3XJiyFtLnYrP4DpM3U="},
{"animals", "2021-10-16", "sha256:hcv2i7awT6CcSCecw9WrYBokFyzYNVaQArGgqHqdj7s="},
}

Expand Down Expand Up @@ -126,14 +126,20 @@ func TestScraper(t *testing.T) {

// Version digests now known to storage
for _, test := range tests {
ok, err := st.HasVersion(ctx, test.name, test.version, test.digest)
c.Assert(err, qt.IsNil)
c.Assert(ok, qt.IsTrue)
if scraper.IsExperimentalVersion(test.version) {
ok, err := st.HasVersion(ctx, test.name, test.version, test.digest)
c.Assert(err, qt.IsNil)
c.Assert(ok, qt.IsFalse, qt.Commentf("experimental version %s should not be included", test.version))
} else {
ok, err := st.HasVersion(ctx, test.name, test.version, test.digest)
c.Assert(err, qt.IsNil)
c.Assert(ok, qt.IsTrue)
}
}

vi, err := st.VersionIndex(ctx)
c.Assert(err, qt.IsNil)
c.Assert(len(vi.Versions()), qt.Equals, 4)
c.Assert(len(vi.Versions()), qt.Equals, 3)
for _, version := range vi.Versions() {
specData, err := st.Version(ctx, version.String())
c.Assert(err, qt.IsNil)
Expand Down

0 comments on commit 3b1628b

Please sign in to comment.