From b6b36a9740a50ec9f51243ffa3453f1347befbd4 Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Tue, 2 Jul 2024 15:23:24 +0200 Subject: [PATCH 1/5] Update metrics response (#536) --- .codespellignore | 1 + docs/api.mdx | 30 ++++++++++++++++++++---------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/.codespellignore b/.codespellignore index a90703856..ab316676a 100644 --- a/.codespellignore +++ b/.codespellignore @@ -1 +1,2 @@ carrer +ser diff --git a/docs/api.mdx b/docs/api.mdx index 71f6d4cdb..dfa5b453b 100644 --- a/docs/api.mdx +++ b/docs/api.mdx @@ -285,13 +285,11 @@ Provides status of build against buildId passed as path variable ```json { - serviceInfo: { - id: string, - status: string, - startTime: string, - duration: string, - succeeded: boolean - } + id: string, + status: string, + startTime: string, + duration: string, + succeeded: boolean } ``` @@ -527,7 +525,7 @@ This endpoint is used to retrieve the builds performed by Wave. ```json { - metric: "builds|pulls|fusion", + metric: "builds", count: integer, orgs: { String: integer, @@ -590,7 +588,13 @@ This endpoint is used to get the pulls performed through Wave. ```json { - count: integer + metric: "pulls", + count: integer, + orgs: { + String: integer, + String: integer, + ... + } } ``` @@ -648,7 +652,13 @@ This endpoint is used to get the pulls of Fusion-based containers performed thro ```json { - count: integer + metric: "fusion", + count: integer, + orgs: { + String: integer, + String: integer, + ... + } } ``` From 5e0d32ac9af863ca592becfc8537552d152a4dde Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Tue, 2 Jul 2024 19:50:04 +0200 Subject: [PATCH 2/5] Refactored metrics service (#549) Signed-off-by: munishchouhan --- .../wave/controller/MetricsController.groovy | 18 ++--- ...nstants.groovy => MetricsConstants.groovy} | 2 +- .../metric/impl/MetricsServiceImpl.groovy | 23 +++--- .../model/GetBuildsCountResponse.groovy | 34 --------- .../model/GetFusionPullsCountResponse.groovy | 34 --------- .../metric/model/GetPullsCountResponse.groovy | 34 --------- .../metric/impl/MetricsServiceImplTest.groovy | 73 +++++++++---------- 7 files changed, 56 insertions(+), 162 deletions(-) rename src/main/groovy/io/seqera/wave/service/metric/{MetricConstants.groovy => MetricsConstants.groovy} (97%) delete mode 100644 src/main/groovy/io/seqera/wave/service/metric/model/GetBuildsCountResponse.groovy delete mode 100644 src/main/groovy/io/seqera/wave/service/metric/model/GetFusionPullsCountResponse.groovy delete mode 100644 src/main/groovy/io/seqera/wave/service/metric/model/GetPullsCountResponse.groovy diff --git a/src/main/groovy/io/seqera/wave/controller/MetricsController.groovy b/src/main/groovy/io/seqera/wave/controller/MetricsController.groovy index 2ba867e78..f3125e49b 100644 --- a/src/main/groovy/io/seqera/wave/controller/MetricsController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/MetricsController.groovy @@ -36,11 +36,9 @@ import io.micronaut.security.annotation.Secured import io.micronaut.security.authentication.AuthorizationException import io.micronaut.security.rules.SecurityRule import io.seqera.wave.exception.BadRequestException -import io.seqera.wave.service.metric.MetricConstants +import io.seqera.wave.service.metric.MetricsConstants import io.seqera.wave.service.metric.MetricsService -import io.seqera.wave.service.metric.model.GetBuildsCountResponse -import io.seqera.wave.service.metric.model.GetFusionPullsCountResponse -import io.seqera.wave.service.metric.model.GetPullsCountResponse + import jakarta.inject.Inject import static io.micronaut.http.HttpHeaders.WWW_AUTHENTICATE @@ -65,25 +63,25 @@ class MetricsController { @Get(uri = "/v1alpha2/metrics/builds", produces = MediaType.APPLICATION_JSON) HttpResponse getBuildsMetrics(@Nullable @QueryValue String date, @Nullable @QueryValue String org) { if(!date && !org) - return HttpResponse.ok(metricsService.getAllOrgCount(MetricConstants.PREFIX_BUILDS)) + return HttpResponse.ok(metricsService.getAllOrgCount(MetricsConstants.PREFIX_BUILDS)) validateQueryParams(date) - return HttpResponse.ok(metricsService.getOrgCount(MetricConstants.PREFIX_BUILDS, date, org)) + return HttpResponse.ok(metricsService.getOrgCount(MetricsConstants.PREFIX_BUILDS, date, org)) } @Get(uri = "/v1alpha2/metrics/pulls", produces = MediaType.APPLICATION_JSON) HttpResponse getPullsMetrics(@Nullable @QueryValue String date, @Nullable @QueryValue String org) { if(!date && !org) - return HttpResponse.ok(metricsService.getAllOrgCount(MetricConstants.PREFIX_PULLS)) + return HttpResponse.ok(metricsService.getAllOrgCount(MetricsConstants.PREFIX_PULLS)) validateQueryParams(date) - return HttpResponse.ok(metricsService.getOrgCount(MetricConstants.PREFIX_PULLS, date, org)) + return HttpResponse.ok(metricsService.getOrgCount(MetricsConstants.PREFIX_PULLS, date, org)) } @Get(uri = "/v1alpha2/metrics/fusion/pulls", produces = MediaType.APPLICATION_JSON) HttpResponse getFusionPullsMetrics(@Nullable @QueryValue String date, @Nullable @QueryValue String org) { if(!date && !org) - return HttpResponse.ok(metricsService.getAllOrgCount(MetricConstants.PREFIX_FUSION)) + return HttpResponse.ok(metricsService.getAllOrgCount(MetricsConstants.PREFIX_FUSION)) validateQueryParams(date) - return HttpResponse.ok(metricsService.getOrgCount(MetricConstants.PREFIX_FUSION, date, org)) + return HttpResponse.ok(metricsService.getOrgCount(MetricsConstants.PREFIX_FUSION, date, org)) } diff --git a/src/main/groovy/io/seqera/wave/service/metric/MetricConstants.groovy b/src/main/groovy/io/seqera/wave/service/metric/MetricsConstants.groovy similarity index 97% rename from src/main/groovy/io/seqera/wave/service/metric/MetricConstants.groovy rename to src/main/groovy/io/seqera/wave/service/metric/MetricsConstants.groovy index 95d397616..84fc14c54 100644 --- a/src/main/groovy/io/seqera/wave/service/metric/MetricConstants.groovy +++ b/src/main/groovy/io/seqera/wave/service/metric/MetricsConstants.groovy @@ -23,7 +23,7 @@ package io.seqera.wave.service.metric * * @author Munish Chouhan */ -interface MetricConstants { +interface MetricsConstants { static final public String PREFIX_FUSION = 'fusion' diff --git a/src/main/groovy/io/seqera/wave/service/metric/impl/MetricsServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/metric/impl/MetricsServiceImpl.groovy index d08e404d5..01c2950c3 100644 --- a/src/main/groovy/io/seqera/wave/service/metric/impl/MetricsServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/metric/impl/MetricsServiceImpl.groovy @@ -25,13 +25,14 @@ import java.util.regex.Pattern import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import io.seqera.wave.service.metric.MetricConstants import io.seqera.wave.service.metric.MetricsCounterStore import io.seqera.wave.service.metric.MetricsService import io.seqera.wave.service.metric.model.GetOrgCountResponse import io.seqera.wave.tower.PlatformId import jakarta.inject.Inject import jakarta.inject.Singleton + +import static io.seqera.wave.service.metric.MetricsConstants.* /** * Implements service to store and retrieve wave metrics from the counter store * @@ -52,13 +53,13 @@ class MetricsServiceImpl implements MetricsService { @Override GetOrgCountResponse getAllOrgCount(String metric){ final response = new GetOrgCountResponse(metric, 0, [:]) - final orgCounts = metricsCounterStore.getAllMatchingEntries("$metric/$MetricConstants.PREFIX_ORG/*") + final orgCounts = metricsCounterStore.getAllMatchingEntries("$metric/$PREFIX_ORG/*") for(def entry : orgCounts) { // orgCounts also contains the records with org and date, so here it filter out the records with date - if(!entry.key.contains("/$MetricConstants.PREFIX_DAY/")) { + if(!entry.key.contains("/$PREFIX_DAY/")) { response.count += entry.value //split is used to extract the org name from the key like "metrics/o/seqera.io" => seqera.io - response.orgs.put(entry.key.split("/$MetricConstants.PREFIX_ORG/").last(), entry.value) + response.orgs.put(entry.key.split("/$PREFIX_ORG/").last(), entry.value) } } return response @@ -76,7 +77,7 @@ class MetricsServiceImpl implements MetricsService { response.orgs.put(org, response.count) }else{ // when only date is provide, scan the store and return the count for all orgs on given date - final orgCounts = metricsCounterStore.getAllMatchingEntries("$metric/$MetricConstants.PREFIX_ORG/*/$MetricConstants.PREFIX_DAY/$date") + final orgCounts = metricsCounterStore.getAllMatchingEntries("$metric/$PREFIX_ORG/*/$PREFIX_DAY/$date") for(def entry : orgCounts) { response.orgs.put(extractOrgFromKey(entry.key), entry.value) } @@ -88,17 +89,17 @@ class MetricsServiceImpl implements MetricsService { @Override void incrementFusionPullsCounter(PlatformId platformId){ - incrementCounter(MetricConstants.PREFIX_FUSION, platformId?.user?.email) + incrementCounter(PREFIX_FUSION, platformId?.user?.email) } @Override void incrementBuildsCounter(PlatformId platformId){ - incrementCounter(MetricConstants.PREFIX_BUILDS, platformId?.user?.email) + incrementCounter(PREFIX_BUILDS, platformId?.user?.email) } @Override void incrementPullsCounter(PlatformId platformId) { - incrementCounter(MetricConstants.PREFIX_PULLS, platformId?.user?.email) + incrementCounter(PREFIX_PULLS, platformId?.user?.email) } protected void incrementCounter(String prefix, String email) { @@ -127,13 +128,13 @@ class MetricsServiceImpl implements MetricsService { protected static String getKey(String prefix, String day, String org){ if( day && org ) - return "$prefix/$MetricConstants.PREFIX_ORG/$org/$MetricConstants.PREFIX_DAY/$day" + return "$prefix/$PREFIX_ORG/$org/$PREFIX_DAY/$day" if( org ) - return "$prefix/$MetricConstants.PREFIX_ORG/$org" + return "$prefix/$PREFIX_ORG/$org" if( day ) - return "$prefix/$MetricConstants.PREFIX_DAY/$day" + return "$prefix/$PREFIX_DAY/$day" return null } diff --git a/src/main/groovy/io/seqera/wave/service/metric/model/GetBuildsCountResponse.groovy b/src/main/groovy/io/seqera/wave/service/metric/model/GetBuildsCountResponse.groovy deleted file mode 100644 index c724c9424..000000000 --- a/src/main/groovy/io/seqera/wave/service/metric/model/GetBuildsCountResponse.groovy +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.metric.model - -import groovy.transform.CompileStatic -/** - * Model a Wave builds count response - * - * @author Munish Chouhan - */ -@CompileStatic -class GetBuildsCountResponse { - Long count - - GetBuildsCountResponse(Long count) { - this.count = count - } -} diff --git a/src/main/groovy/io/seqera/wave/service/metric/model/GetFusionPullsCountResponse.groovy b/src/main/groovy/io/seqera/wave/service/metric/model/GetFusionPullsCountResponse.groovy deleted file mode 100644 index 1c9659111..000000000 --- a/src/main/groovy/io/seqera/wave/service/metric/model/GetFusionPullsCountResponse.groovy +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.metric.model - -import groovy.transform.CompileStatic -/** - * Model a Wave fusion pulls count response - * - * @author Munish Chouhan - */ -@CompileStatic -class GetFusionPullsCountResponse { - Long count - - GetFusionPullsCountResponse(Long count) { - this.count = count - } -} diff --git a/src/main/groovy/io/seqera/wave/service/metric/model/GetPullsCountResponse.groovy b/src/main/groovy/io/seqera/wave/service/metric/model/GetPullsCountResponse.groovy deleted file mode 100644 index 3de656ac7..000000000 --- a/src/main/groovy/io/seqera/wave/service/metric/model/GetPullsCountResponse.groovy +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.metric.model - -import groovy.transform.CompileStatic -/** - * Model a Wave pulls count response - * - * @author Munish Chouhan - */ -@CompileStatic -class GetPullsCountResponse { - Long count - - GetPullsCountResponse(Long count) { - this.count = count - } -} diff --git a/src/test/groovy/io/seqera/wave/service/metric/impl/MetricsServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/metric/impl/MetricsServiceImplTest.groovy index 23c9335f1..87389577e 100644 --- a/src/test/groovy/io/seqera/wave/service/metric/impl/MetricsServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/metric/impl/MetricsServiceImplTest.groovy @@ -25,15 +25,12 @@ import java.time.LocalDate import java.time.format.DateTimeFormatter import io.seqera.wave.service.counter.impl.LocalCounterProvider -import io.seqera.wave.service.counter.impl.RedisCounterProvider -import io.seqera.wave.service.metric.MetricConstants import io.seqera.wave.service.metric.MetricsCounterStore import io.seqera.wave.test.RedisTestContainer import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.User -import org.testcontainers.shaded.org.bouncycastle.cms.OriginatorInfoGenerator -import software.amazon.awssdk.regions.servicemetadata.OrganizationsServiceMetadata +import static io.seqera.wave.service.metric.MetricsConstants.* /** * * @author Munish Chouhan @@ -59,19 +56,19 @@ class MetricsServiceImplTest extends Specification implements RedisTestContainer metricsService.incrementBuildsCounter(null) then: - def res1 = metricsService.getOrgCount(MetricConstants.PREFIX_BUILDS, date, null) + def res1 = metricsService.getOrgCount(PREFIX_BUILDS, date, null) res1.count == 3 res1.orgs == ['org1.com': 1, 'org2.com': 1] and: - def res2 = metricsService.getOrgCount(MetricConstants.PREFIX_BUILDS, null, 'org1.com') + def res2 = metricsService.getOrgCount(PREFIX_BUILDS, null, 'org1.com') res2.count == 1 res2.orgs == ['org1.com': 1] and: - def res3 = metricsService.getOrgCount(MetricConstants.PREFIX_BUILDS, date, 'org2.com') + def res3 = metricsService.getOrgCount(PREFIX_BUILDS, date, 'org2.com') res3.count == 1 res3.orgs == ['org2.com': 1] and: - def res4 = metricsService.getOrgCount(MetricConstants.PREFIX_BUILDS, date, null) + def res4 = metricsService.getOrgCount(PREFIX_BUILDS, date, null) res4.count == 3 res4.orgs == ['org1.com': 1, 'org2.com': 1] } @@ -93,15 +90,15 @@ class MetricsServiceImplTest extends Specification implements RedisTestContainer metricsService.incrementPullsCounter(null) then: - def res1 = metricsService.getOrgCount(MetricConstants.PREFIX_PULLS, null, 'org1.com') + def res1 = metricsService.getOrgCount(PREFIX_PULLS, null, 'org1.com') res1.count == 1 res1.orgs == ['org1.com': 1] and: - def res2 = metricsService.getOrgCount(MetricConstants.PREFIX_PULLS,date, 'org2.com') + def res2 = metricsService.getOrgCount(PREFIX_PULLS,date, 'org2.com') res2.count == 1 res2.orgs == ['org2.com': 1] and: - def res3 = metricsService.getOrgCount(MetricConstants.PREFIX_PULLS,date, null) + def res3 = metricsService.getOrgCount(PREFIX_PULLS,date, null) res3.count == 3 res3.orgs == ['org1.com': 1, 'org2.com': 1] } @@ -123,15 +120,15 @@ class MetricsServiceImplTest extends Specification implements RedisTestContainer metricsService.incrementFusionPullsCounter(null) then: - def res1 = metricsService.getOrgCount(MetricConstants.PREFIX_FUSION,null, 'org1.com') + def res1 = metricsService.getOrgCount(PREFIX_FUSION,null, 'org1.com') res1.count == 1 res1.orgs == ['org1.com': 1] and: - def res2 = metricsService.getOrgCount(MetricConstants.PREFIX_FUSION, date, 'org2.com') + def res2 = metricsService.getOrgCount(PREFIX_FUSION, date, 'org2.com') res2.count == 1 res2.orgs == ['org2.com': 1] and: - def res3 = metricsService.getOrgCount(MetricConstants.PREFIX_FUSION, date, null) + def res3 = metricsService.getOrgCount(PREFIX_FUSION, date, null) res3.count == 3 res3.orgs == ['org1.com': 1, 'org2.com': 1] } @@ -143,18 +140,18 @@ class MetricsServiceImplTest extends Specification implements RedisTestContainer where: PREFIX | DAY | ORG | KEY - MetricConstants.PREFIX_BUILDS | null | null | null - MetricConstants.PREFIX_BUILDS | null | 'wave' | 'builds/o/wave' - MetricConstants.PREFIX_BUILDS | '2024-03-25' | 'wave' | 'builds/o/wave/d/2024-03-25' - MetricConstants.PREFIX_BUILDS | '2024-03-25' | null | 'builds/d/2024-03-25' - MetricConstants.PREFIX_PULLS | null | null | null - MetricConstants.PREFIX_PULLS | null | 'wave' | 'pulls/o/wave' - MetricConstants.PREFIX_PULLS | '2024-03-25' | 'wave' | 'pulls/o/wave/d/2024-03-25' - MetricConstants.PREFIX_PULLS | '2024-03-25' | null | 'pulls/d/2024-03-25' - MetricConstants.PREFIX_FUSION | null | null | null - MetricConstants.PREFIX_FUSION | null | 'wave' | 'fusion/o/wave' - MetricConstants.PREFIX_FUSION | '2024-03-25' | 'wave' | 'fusion/o/wave/d/2024-03-25' - MetricConstants.PREFIX_FUSION | '2024-03-25' | null | 'fusion/d/2024-03-25' + PREFIX_BUILDS | null | null | null + PREFIX_BUILDS | null | 'wave' | 'builds/o/wave' + PREFIX_BUILDS | '2024-03-25' | 'wave' | 'builds/o/wave/d/2024-03-25' + PREFIX_BUILDS | '2024-03-25' | null | 'builds/d/2024-03-25' + PREFIX_PULLS | null | null | null + PREFIX_PULLS | null | 'wave' | 'pulls/o/wave' + PREFIX_PULLS | '2024-03-25' | 'wave' | 'pulls/o/wave/d/2024-03-25' + PREFIX_PULLS | '2024-03-25' | null | 'pulls/d/2024-03-25' + PREFIX_FUSION | null | null | null + PREFIX_FUSION | null | 'wave' | 'fusion/o/wave' + PREFIX_FUSION | '2024-03-25' | 'wave' | 'fusion/o/wave/d/2024-03-25' + PREFIX_FUSION | '2024-03-25' | null | 'fusion/d/2024-03-25' } @Unroll @@ -192,21 +189,21 @@ class MetricsServiceImplTest extends Specification implements RedisTestContainer metricsService.incrementFusionPullsCounter(platformId2) metricsService.incrementFusionPullsCounter(null) and: - def buildOrgCounts = metricsService.getAllOrgCount(MetricConstants.PREFIX_BUILDS) - def pullOrgCounts = metricsService.getAllOrgCount(MetricConstants.PREFIX_PULLS) - def fusionOrgCounts = metricsService.getAllOrgCount(MetricConstants.PREFIX_FUSION) + def buildOrgCounts = metricsService.getAllOrgCount(PREFIX_BUILDS) + def pullOrgCounts = metricsService.getAllOrgCount(PREFIX_PULLS) + def fusionOrgCounts = metricsService.getAllOrgCount(PREFIX_FUSION) def emptyOrgCounts = metricsService.getAllOrgCount(null) then: - buildOrgCounts.metric == MetricConstants.PREFIX_BUILDS + buildOrgCounts.metric == PREFIX_BUILDS buildOrgCounts.count == 2 buildOrgCounts.orgs == ['org1.com': 1, 'org2.com': 1] and: - pullOrgCounts.metric == MetricConstants.PREFIX_PULLS + pullOrgCounts.metric == PREFIX_PULLS pullOrgCounts.count == 2 pullOrgCounts.orgs == ['org1.com': 1, 'org2.com': 1] and: - fusionOrgCounts.metric == MetricConstants.PREFIX_FUSION + fusionOrgCounts.metric == PREFIX_FUSION fusionOrgCounts.count == 2 fusionOrgCounts.orgs == ['org1.com': 1, 'org2.com': 1] and: @@ -237,21 +234,21 @@ class MetricsServiceImplTest extends Specification implements RedisTestContainer metricsService.incrementFusionPullsCounter(platformId2) metricsService.incrementFusionPullsCounter(null) and: - def buildOrgCounts = metricsService.getOrgCount(MetricConstants.PREFIX_BUILDS, date, null) - def pullOrgCounts = metricsService.getOrgCount(MetricConstants.PREFIX_PULLS, date, null) - def fusionOrgCounts = metricsService.getOrgCount(MetricConstants.PREFIX_FUSION, date, null) + def buildOrgCounts = metricsService.getOrgCount(PREFIX_BUILDS, date, null) + def pullOrgCounts = metricsService.getOrgCount(PREFIX_PULLS, date, null) + def fusionOrgCounts = metricsService.getOrgCount(PREFIX_FUSION, date, null) def emptyOrgCounts = metricsService.getOrgCount(null, date, null) then: - buildOrgCounts.metric == MetricConstants.PREFIX_BUILDS + buildOrgCounts.metric == PREFIX_BUILDS buildOrgCounts.count == 3 buildOrgCounts.orgs == ['org1.com': 1, 'org2.com': 1] and: - pullOrgCounts.metric == MetricConstants.PREFIX_PULLS + pullOrgCounts.metric == PREFIX_PULLS pullOrgCounts.count == 3 pullOrgCounts.orgs == ['org1.com': 1, 'org2.com': 1] and: - fusionOrgCounts.metric == MetricConstants.PREFIX_FUSION + fusionOrgCounts.metric == PREFIX_FUSION fusionOrgCounts.count == 3 fusionOrgCounts.orgs == ['org1.com': 1, 'org2.com': 1] and: From 32f7dd1672b43c0e4ce1a9e2f4dac714eceeda47 Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Wed, 3 Jul 2024 10:48:55 +0200 Subject: [PATCH 3/5] Add Typespec API definitions (#537) Signed-off-by: munishchouhan Co-authored-by: Phil Ewels --- .github/workflows/typespec.yml | 40 ++++++++ .gitignore | 5 + README.md | 11 +++ typespec/main.tsp | 4 + typespec/models/BuildStatusResponse.tsp | 8 ++ typespec/models/CondaOpts.tsp | 6 ++ typespec/models/ContainerConfig.tsp | 10 ++ typespec/models/ContainerInspectConfig.tsp | 18 ++++ typespec/models/ContainerInspectRequest.tsp | 7 ++ typespec/models/ContainerInspectResponse.tsp | 18 ++++ typespec/models/ContainerLayer.tsp | 8 ++ typespec/models/ContainerRequest.tsp | 26 +++++ typespec/models/ContainerResponse.tsp | 10 ++ typespec/models/Manifest.tsp | 13 +++ typespec/models/ManifestLayer.tsp | 6 ++ typespec/models/MetricsResponse.tsp | 11 +++ typespec/models/Packages.tsp | 12 +++ typespec/models/RootFS.tsp | 5 + typespec/models/SpackOpts.tsp | 5 + .../models/ValidateRegistryCredsRequest.tsp | 7 ++ typespec/models/Vulnerability.tsp | 10 ++ typespec/models/WaveBuildRecord.tsp | 21 ++++ typespec/models/WaveScanRecord.tsp | 11 +++ typespec/models/models.tsp | 9 ++ typespec/package.json | 12 +++ typespec/routes.tsp | 98 +++++++++++++++++++ typespec/tspconfig.yaml | 2 + 27 files changed, 393 insertions(+) create mode 100644 .github/workflows/typespec.yml create mode 100644 typespec/main.tsp create mode 100644 typespec/models/BuildStatusResponse.tsp create mode 100644 typespec/models/CondaOpts.tsp create mode 100644 typespec/models/ContainerConfig.tsp create mode 100644 typespec/models/ContainerInspectConfig.tsp create mode 100644 typespec/models/ContainerInspectRequest.tsp create mode 100644 typespec/models/ContainerInspectResponse.tsp create mode 100644 typespec/models/ContainerLayer.tsp create mode 100644 typespec/models/ContainerRequest.tsp create mode 100644 typespec/models/ContainerResponse.tsp create mode 100644 typespec/models/Manifest.tsp create mode 100644 typespec/models/ManifestLayer.tsp create mode 100644 typespec/models/MetricsResponse.tsp create mode 100644 typespec/models/Packages.tsp create mode 100644 typespec/models/RootFS.tsp create mode 100644 typespec/models/SpackOpts.tsp create mode 100644 typespec/models/ValidateRegistryCredsRequest.tsp create mode 100644 typespec/models/Vulnerability.tsp create mode 100644 typespec/models/WaveBuildRecord.tsp create mode 100644 typespec/models/WaveScanRecord.tsp create mode 100644 typespec/models/models.tsp create mode 100644 typespec/package.json create mode 100644 typespec/routes.tsp create mode 100644 typespec/tspconfig.yaml diff --git a/.github/workflows/typespec.yml b/.github/workflows/typespec.yml new file mode 100644 index 000000000..8d9adaa33 --- /dev/null +++ b/.github/workflows/typespec.yml @@ -0,0 +1,40 @@ +--- +name: Typespec_Validation + +on: + push: + branches: + - '**' + paths : + - 'typespec/**' + pull_request: + types: [opened, reopened, synchronize] + paths: + - 'typespec/**' + +permissions: + contents: read + +jobs: + typespec_validation: + name: validate typespec files + runs-on: ubuntu-latest + + steps: + - name : Checkout + uses : actions/checkout@v4 + + - name : Setup Node.js environment + uses : actions/setup-node@v4 + with : + node-version : '20.9.0' + + - name : Install tsp + run : npm install -g @typespec/compiler + + - name : Validate tsp files + run : | + cd typespec + tsp install + tsp compile . + diff --git a/.gitignore b/.gitignore index e44a57f7f..41b2ff645 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,8 @@ scan-workspace/ .cache site/ deployment-url.txt + +#typespec +tsp-output/ +node_modules/ +package-lock.json diff --git a/README.md b/README.md index 070ea0d4a..da025233e 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,17 @@ container registry where the image is stored, while the instrumented layers are '-Djdk.httpclient.HttpClient.log=requests,headers' ``` +## TypeSpec API Specifications + +- You can find the API specifications using (typespec)[https://github.com/microsoft/typespec] in typespec directory. Use following command to generate the API specifications. + + ```bash + 'cd typespec' + 'tsp install' + 'tsp compile .' + ``` + +- Check `typespec/tsp-output` directory for the generated API specifications. ## Related links * [Wave command line tool](https://github.com/seqeralabs/wave-cli) diff --git a/typespec/main.tsp b/typespec/main.tsp new file mode 100644 index 000000000..6633a9e30 --- /dev/null +++ b/typespec/main.tsp @@ -0,0 +1,4 @@ +import "@typespec/http"; +import "@typespec/rest"; +import "@typespec/openapi3"; +import "./routes.tsp"; diff --git a/typespec/models/BuildStatusResponse.tsp b/typespec/models/BuildStatusResponse.tsp new file mode 100644 index 000000000..18da9ea1c --- /dev/null +++ b/typespec/models/BuildStatusResponse.tsp @@ -0,0 +1,8 @@ +@doc("Response payload for build status.") +model BuildStatusResponse { + duration: string; + id: string; + startTime: string; + status: "PENDING" | "COMPLETED"; + succeeded: boolean; +} diff --git a/typespec/models/CondaOpts.tsp b/typespec/models/CondaOpts.tsp new file mode 100644 index 000000000..f1c94356a --- /dev/null +++ b/typespec/models/CondaOpts.tsp @@ -0,0 +1,6 @@ +@doc("Options for Conda environments.") +model CondaOpts { + basePackages: string; + commands: string[]; + mambaImage: string; +} \ No newline at end of file diff --git a/typespec/models/ContainerConfig.tsp b/typespec/models/ContainerConfig.tsp new file mode 100644 index 000000000..834297775 --- /dev/null +++ b/typespec/models/ContainerConfig.tsp @@ -0,0 +1,10 @@ +import "./ContainerLayer.tsp"; + +@doc("Configuration details for a container.") +model ContainerConfig { + cmd: string[]; + entrypoint: string[]; + env: string[]; + layers: ContainerLayer[]; + workingDir: string; +} diff --git a/typespec/models/ContainerInspectConfig.tsp b/typespec/models/ContainerInspectConfig.tsp new file mode 100644 index 000000000..bf59c40e9 --- /dev/null +++ b/typespec/models/ContainerInspectConfig.tsp @@ -0,0 +1,18 @@ +import "./RootFS.tsp"; + +@doc("Configuration details of a container.") +model Config { + architecture: string; + config: { + attachStdin: boolean; + attachStdout: boolean; + attachStderr: boolean; + tty: boolean; + env: string[]; + cmd: string[]; + image: string; + }; + container: string; + created: string; + rootfs: RootFS; +} \ No newline at end of file diff --git a/typespec/models/ContainerInspectRequest.tsp b/typespec/models/ContainerInspectRequest.tsp new file mode 100644 index 000000000..00e2dd23e --- /dev/null +++ b/typespec/models/ContainerInspectRequest.tsp @@ -0,0 +1,7 @@ +@doc("Request payload for inspecting a container.") +model ContainerInspectRequest { + containerImage: string; + towerAccessToken: string; + towerEndpoint: string; + towerWorkspaceId: int64; +} \ No newline at end of file diff --git a/typespec/models/ContainerInspectResponse.tsp b/typespec/models/ContainerInspectResponse.tsp new file mode 100644 index 000000000..6aee1cb04 --- /dev/null +++ b/typespec/models/ContainerInspectResponse.tsp @@ -0,0 +1,18 @@ +import "./ContainerInspectConfig.tsp"; +import "./Manifest.tsp"; + +@doc("Response payload for inspecting a container.") +model ContainerInspectResponse { + Container: { + registry: string; + hostName: string; + imageName: string; + reference: string; + digest: string; + config: Config; + manifest: Manifest; + v1: boolean; + v2: boolean; + oci: boolean; + } +} diff --git a/typespec/models/ContainerLayer.tsp b/typespec/models/ContainerLayer.tsp new file mode 100644 index 000000000..873cb4c84 --- /dev/null +++ b/typespec/models/ContainerLayer.tsp @@ -0,0 +1,8 @@ +@doc("Represents a layer in a container image.") +model ContainerLayer { + gzipDigest: string; + gzipSize: string; + location: string; + skipHashing: boolean; + tarDigest: string; +} diff --git a/typespec/models/ContainerRequest.tsp b/typespec/models/ContainerRequest.tsp new file mode 100644 index 000000000..666feee43 --- /dev/null +++ b/typespec/models/ContainerRequest.tsp @@ -0,0 +1,26 @@ +import "./ContainerConfig.tsp"; +import "./Packages.tsp"; + +@doc("Request payload for creating a container token.") +model ContainerRequest { + buildContext: ContainerLayer; + buildRepository?: string; + cacheRepository?: string; + containerConfig: ContainerConfig; + containerFile?: string; + containerImage: string; + containerIncludes: string[]; + containerPlatform: string; + dryRun: boolean; + fingerprint?: string; + format: "sif" | "docker"; + freeze?: boolean; + nameStrategy?: "none" | "tagPrefix" | "imageSuffix"; + packages?: Packages; + timestamp: string; + towerAccessToken?: string; + towerEndpoint?: string; + towerRefreshToken?: string; + towerWorkspaceId?: int32; + workflowId: string; +} diff --git a/typespec/models/ContainerResponse.tsp b/typespec/models/ContainerResponse.tsp new file mode 100644 index 000000000..649d64048 --- /dev/null +++ b/typespec/models/ContainerResponse.tsp @@ -0,0 +1,10 @@ +@doc("Response payload for container token creation.") +model ContainerResponse { + buildId: string; + cached: boolean; + containerImage: string; + containerToken: string; + expiration: string; + freeze?: boolean; + targetImage: string; +} \ No newline at end of file diff --git a/typespec/models/Manifest.tsp b/typespec/models/Manifest.tsp new file mode 100644 index 000000000..60f93bcc2 --- /dev/null +++ b/typespec/models/Manifest.tsp @@ -0,0 +1,13 @@ +import "./ManifestLayer.tsp"; + +@doc("Manifest details of a container.") +model Manifest { + config: { + digest: string; + mediaType: string; + size: int64; + }; + layers: ManifestLayer[]; + mediaType: string; + schemaVersion: int32; +} \ No newline at end of file diff --git a/typespec/models/ManifestLayer.tsp b/typespec/models/ManifestLayer.tsp new file mode 100644 index 000000000..418904eb9 --- /dev/null +++ b/typespec/models/ManifestLayer.tsp @@ -0,0 +1,6 @@ +@doc("Manifest layer details of a container.") +model ManifestLayer { + digest: string; + mediaType: string; + size: int64; +} \ No newline at end of file diff --git a/typespec/models/MetricsResponse.tsp b/typespec/models/MetricsResponse.tsp new file mode 100644 index 000000000..f17da0d26 --- /dev/null +++ b/typespec/models/MetricsResponse.tsp @@ -0,0 +1,11 @@ +@doc("Response payload for metrics.") +model MetricsResponse { + count: int64; + metric: "builds" | "fusion" | "pulls"; + orgs: Orgs; +} + +model Orgs { + key: string; + value: int64; +} \ No newline at end of file diff --git a/typespec/models/Packages.tsp b/typespec/models/Packages.tsp new file mode 100644 index 000000000..6fa59e16d --- /dev/null +++ b/typespec/models/Packages.tsp @@ -0,0 +1,12 @@ +import "./CondaOpts.tsp"; +import "./SpackOpts.tsp"; + +@doc("Package configurations for container builds.") +model Packages { + channels: string[]; + condaOpts?: CondaOpts; + entries: string[]; + environment: string; + spackOpts?: SpackOpts; + type: "CONDA" | "SPACK"; +} \ No newline at end of file diff --git a/typespec/models/RootFS.tsp b/typespec/models/RootFS.tsp new file mode 100644 index 000000000..2ec6be063 --- /dev/null +++ b/typespec/models/RootFS.tsp @@ -0,0 +1,5 @@ +@doc("Details about the root filesystem of a container.") +model RootFS { + diff_ids: string[]; + type: string; +} diff --git a/typespec/models/SpackOpts.tsp b/typespec/models/SpackOpts.tsp new file mode 100644 index 000000000..cc7a9ea2e --- /dev/null +++ b/typespec/models/SpackOpts.tsp @@ -0,0 +1,5 @@ +@doc("Options for Spack environments.") +model SpackOpts { + basePackages: string; + commands: string[]; +} \ No newline at end of file diff --git a/typespec/models/ValidateRegistryCredsRequest.tsp b/typespec/models/ValidateRegistryCredsRequest.tsp new file mode 100644 index 000000000..c1ef4ba95 --- /dev/null +++ b/typespec/models/ValidateRegistryCredsRequest.tsp @@ -0,0 +1,7 @@ +@doc("request payload of validate credentials request") +model ValidateRegistryCredsRequest { + password: string; + registry: string; + userName: string; + } + \ No newline at end of file diff --git a/typespec/models/Vulnerability.tsp b/typespec/models/Vulnerability.tsp new file mode 100644 index 000000000..9a4e5339c --- /dev/null +++ b/typespec/models/Vulnerability.tsp @@ -0,0 +1,10 @@ +@doc("Scan Vulnerability details") +model Vulnerability { + fixedVersion: string; + id: string; + installedVersion: string; + pkgName: string; + primaryUrl: string; + severity: string; + title: string; + } \ No newline at end of file diff --git a/typespec/models/WaveBuildRecord.tsp b/typespec/models/WaveBuildRecord.tsp new file mode 100644 index 000000000..f8e40bc4c --- /dev/null +++ b/typespec/models/WaveBuildRecord.tsp @@ -0,0 +1,21 @@ +model WaveBuildRecord { + buildId: string; + condaFile: string; + digest: string; + dockerFile: string; + duration: int64; + exitStatus: int32; + format: "docker" | "sif"; + offsetId: string; + platform: string; + requestIp: string; + scanId: string; + spackFile: string; + startTime: string; + succeeded: boolean; + targetImage: string; + userEmail: string; + userId: int64; + userName: string; + } + \ No newline at end of file diff --git a/typespec/models/WaveScanRecord.tsp b/typespec/models/WaveScanRecord.tsp new file mode 100644 index 000000000..e74e56cf9 --- /dev/null +++ b/typespec/models/WaveScanRecord.tsp @@ -0,0 +1,11 @@ +import "./Vulnerability.tsp"; + +@doc("Response Payload for wave scan") +model WaveScanRecord { + buildId: string; + duration: int64; + id: string; + startTime: string; + status: string; + vulnerabilities: Vulnerability[]; + } \ No newline at end of file diff --git a/typespec/models/models.tsp b/typespec/models/models.tsp new file mode 100644 index 000000000..c63c2e6f3 --- /dev/null +++ b/typespec/models/models.tsp @@ -0,0 +1,9 @@ +import "./ContainerRequest.tsp"; +import "./ContainerResponse.tsp"; +import "./BuildStatusResponse.tsp"; +import "./ContainerInspectRequest.tsp"; +import "./ContainerInspectResponse.tsp"; +import "./MetricsResponse.tsp"; +import "./WaveScanRecord.tsp"; +import "./WaveBuildRecord.tsp"; +import "./ValidateRegistryCredsRequest.tsp"; \ No newline at end of file diff --git a/typespec/package.json b/typespec/package.json new file mode 100644 index 000000000..5e792742d --- /dev/null +++ b/typespec/package.json @@ -0,0 +1,12 @@ +{ + "name": "wave", + "version": "1.8.2", + "type": "module", + "dependencies": { + "@typespec/compiler": "latest", + "@typespec/http": "latest", + "@typespec/rest": "latest", + "@typespec/openapi3": "latest" + }, + "private": true +} \ No newline at end of file diff --git a/typespec/routes.tsp b/typespec/routes.tsp new file mode 100644 index 000000000..d3d1c7623 --- /dev/null +++ b/typespec/routes.tsp @@ -0,0 +1,98 @@ +import "./models/models.tsp"; + +using TypeSpec.Http; +using TypeSpec.Rest; + +@service({ + title: "Wave service", +}) +@server("https://wave.seqera.io", "wave endopint") +namespace wave { + @route("/v1alpha2/container") + interface ContainerService { + + @post op createV1Alpha2Container(@body requestBody: ContainerRequest): { + @body response: ContainerResponse; + @statusCode statusCode: 200; + }; + + } + + @route("/v1alpha1/builds/{buildId}") + interface BuildService { + + @get op getBuildRecord(@path buildId: string): { + @body response: WaveBuildRecord; + @statusCode statusCode: 200; + }|{ + @statusCode statusCode: 404; + }; + + @route("/status") + @get op getBuildStatus(@path buildId: string): { + @body response: BuildStatusResponse; + @statusCode statusCode: 200; + }|{ + @statusCode statusCode: 404; + }; + + @route("/logs") + @get op getBuildLogs(@path buildId: string): { + @body response: string; + @statusCode statusCode: 200; + }|{ + @statusCode statusCode: 404; + }; + + } + + @route("/v1alpha1/scans/{scanId}") + interface scanService{ + + @get op scanImage(@path scanId: string) : { + @body response: WaveScanRecord; + @statusCode statusCode: 200; + }|{ + @statusCode statusCode: 404; + }; + + } + + @route("/v1alpha1/inspect") + interface InspectService { + + @post op inspectContainer(@body requestBody: ContainerInspectRequest): { + @body response: ContainerInspectResponse; + @statusCode statusCode: 200; + }|{ + @statusCode statusCode: 404; + }; + + } + + @route("/v1alpha2/metrics") + interface MetricsService { + + @route("/builds") + @get op getBuildMetrics(@query date?: string, @query org?: string): { + @body response: MetricsResponse; + @statusCode statusCode: 200; + }; + + @route("/pulls") + @get op getPullMetrics(@query date?: string, @query org?: string): { + @body response: MetricsResponse; + @statusCode statusCode: 200; + }; + + @route("/fusion/pulls") + @get op getFusionPullMetrics(@query date?: string, @query org?: string): { + @body response: MetricsResponse; + @statusCode statusCode: 200; + }; + } + + @route("validate-creds") + @post op validateCreds(@body request: ValidateRegistryCredsRequest): boolean; + +} diff --git a/typespec/tspconfig.yaml b/typespec/tspconfig.yaml new file mode 100644 index 000000000..a3fe48f13 --- /dev/null +++ b/typespec/tspconfig.yaml @@ -0,0 +1,2 @@ +emit: + - "@typespec/openapi3" From b0c775a3cf378ad4e325b3a43f435107686ecb5c Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Thu, 4 Jul 2024 16:14:23 +0200 Subject: [PATCH 4/5] Check and delete corrupted blobs cache uploads (#533) Signed-off-by: munishchouhan Signed-off-by: Paolo Di Tommaso Co-authored-by: Paolo Di Tommaso --- .../wave/configuration/BlobCacheConfig.groovy | 3 + .../blob/impl/BlobCacheServiceImpl.groovy | 82 +++++++++++- .../service/blob/impl/S3ClientFactory.groovy | 62 +++++++++ .../blob/impl/BlobCacheServiceImplTest.groovy | 119 ++++++++++++++++++ .../impl/BlobCacheServiceImplTest2.groovy | 9 +- .../wave/test/AwsS3TestContainer.groovy | 3 +- 6 files changed, 267 insertions(+), 11 deletions(-) create mode 100644 src/main/groovy/io/seqera/wave/service/blob/impl/S3ClientFactory.groovy diff --git a/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy index 13935e6c0..fe33b40d6 100644 --- a/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy @@ -41,6 +41,9 @@ class BlobCacheConfig { @Value('${wave.blobCache.status.delay:5s}') Duration statusDelay + @Value('${wave.blobCache.failure.duration:4s}') + Duration failureDuration + @Value('${wave.blobCache.timeout:5m}') Duration transferTimeout diff --git a/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy index 51c8fb3cf..6f8f91b05 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy @@ -20,6 +20,7 @@ package io.seqera.wave.service.blob.impl import java.net.http.HttpClient import java.net.http.HttpRequest import java.net.http.HttpResponse +import java.util.concurrent.CompletableFuture import java.util.concurrent.ExecutorService import groovy.transform.CompileStatic @@ -38,6 +39,7 @@ import io.seqera.wave.service.blob.BlobSigningService import io.seqera.wave.service.blob.BlobStore import io.seqera.wave.service.blob.TransferStrategy import io.seqera.wave.service.blob.TransferTimeoutException +import io.seqera.wave.util.BucketTokenizer import io.seqera.wave.util.Escape import io.seqera.wave.util.Retryable import io.seqera.wave.util.StringUtils @@ -45,6 +47,9 @@ import jakarta.annotation.PostConstruct import jakarta.inject.Inject import jakarta.inject.Named import jakarta.inject.Singleton +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest +import software.amazon.awssdk.services.s3.model.HeadObjectRequest import static io.seqera.wave.WaveDefault.HTTP_SERVER_ERRORS /** * Implements cache for container image layer blobs @@ -82,6 +87,10 @@ class BlobCacheServiceImpl implements BlobCacheService { @Inject private HttpClientConfig httpConfig + @Inject + @Named('BlobS3Client') + private S3Client s3Client + private HttpClient httpClient @PostConstruct @@ -184,21 +193,38 @@ class BlobCacheServiceImpl implements BlobCacheService { else { log.debug "== Blob cache begin for object '${info.locationUri}'" result = store(route, info) + //check if the cached blob size is correct + result = checkUploadedBlobSize(result, route) } } finally { // use a short time-to-live for failed downloads - // this is needed to allow re-try downloads failed for - // temporary error conditions e.g. expired credentials + // this is needed to allow re-try caching of failure transfers final ttl = result.succeeded() ? blobConfig.statusDuration - : blobConfig.statusDelay.multipliedBy(10) + : blobConfig.failureDuration blobStore.storeBlob(route.targetPath, result, ttl) return result } } + /** + * Check the size of the blob stored in the cache + * + * @return {@link BlobCacheInfo} the blob cache info + */ + protected BlobCacheInfo checkUploadedBlobSize(BlobCacheInfo info, RoutePath route) { + if( !info.succeeded() ) + return info + final blobSize = getBlobSize(route) + if( blobSize == info.contentLength ) + return info + log.warn("== Blob cache mismatch size for uploaded object '${info.locationUri}'; upload blob size: ${blobSize}; expect size: ${info.contentLength}") + CompletableFuture.supplyAsync(() -> deleteBlob(route), executor) + return info.failed("Mismatch cache size for object ${info.locationUri}") + } + protected BlobCacheInfo store(RoutePath route, BlobCacheInfo info) { final target = route.targetPath try { @@ -226,7 +252,6 @@ class BlobCacheServiceImpl implements BlobCacheService { StringUtils.pathConcat(blobConfig.storageBucket, route.targetPath) } - /** * The HTTP URI from there the cached layer blob is going to be downloaded * @@ -234,7 +259,7 @@ class BlobCacheServiceImpl implements BlobCacheService { * @return The HTTP URI from the cached layer blob is going to be downloaded */ protected String blobDownloadUri(RoutePath route) { - final bucketPath = StringUtils.pathConcat(blobConfig.storageBucket, route.targetPath) + final bucketPath = blobStorePath(route) final presignedUrl = signingService.createSignedUri(bucketPath) if( blobConfig.baseUrl ) { @@ -247,7 +272,6 @@ class BlobCacheServiceImpl implements BlobCacheService { return presignedUrl } - /** * Await for the container layer blob download * @@ -293,4 +317,50 @@ class BlobCacheServiceImpl implements BlobCacheService { } } } + + /** + * get the size of the blob stored in the cache + * + * @return {@link Long} the size of the blob stored in the cache + */ + protected Long getBlobSize(RoutePath route) { + final objectUri = blobStorePath(route) + final object = BucketTokenizer.from(objectUri) + try { + final request = + HeadObjectRequest.builder() + .bucket(object.bucket) + .key(object.key) + .build() + final headObjectResponse = s3Client.headObject(request as HeadObjectRequest) + final contentLength = headObjectResponse.contentLength() + return contentLength!=null ? contentLength : -1L + } + catch (Exception e){ + log.error("== Blob cache Error getting content length of object $objectUri from bucket ${blobConfig.storageBucket}", e) + return -1L + } + } + + /** + * delete the blob stored in the cache + * + */ + protected void deleteBlob(RoutePath route) { + final objectUri = blobStorePath(route) + log.debug "== Blob cache Deleting object $objectUri" + final object = BucketTokenizer.from(objectUri) + try { + final request = + DeleteObjectRequest.builder() + .bucket(object.bucket) + .key(object.key) + .build() + s3Client.deleteObject(request as DeleteObjectRequest) + log.debug("== Blob cache Deleted object $objectUri from bucket ${blobConfig.storageBucket}") + } + catch (Exception e){ + log.error("== Blob cache Error deleting object $objectUri from bucket ${blobConfig.storageBucket}", e) + } + } } diff --git a/src/main/groovy/io/seqera/wave/service/blob/impl/S3ClientFactory.groovy b/src/main/groovy/io/seqera/wave/service/blob/impl/S3ClientFactory.groovy new file mode 100644 index 000000000..af14deba6 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/blob/impl/S3ClientFactory.groovy @@ -0,0 +1,62 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.blob.impl + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Factory +import io.micronaut.context.annotation.Requires +import io.seqera.wave.configuration.BlobCacheConfig +import jakarta.inject.Inject +import jakarta.inject.Named +import jakarta.inject.Singleton +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.s3.S3Client +/** + * Factory implementation for S3Client + * + * @author Munish Chouhan + */ +@Factory +@CompileStatic +@Slf4j +@Requires(property = 'wave.blobCache.enabled', value = 'true') +class S3ClientFactory { + + @Inject + private BlobCacheConfig blobConfig + + @Singleton + @Named('BlobS3Client') + S3Client cloudflareS3Client() { + final creds = AwsBasicCredentials.create(blobConfig.storageAccessKey, blobConfig.storageSecretKey) + final builder = S3Client.builder() + .region(Region.of(blobConfig.storageRegion)) + .credentialsProvider(StaticCredentialsProvider.create(creds)) + + if (blobConfig.storageEndpoint) { + builder.endpointOverride(URI.create(blobConfig.storageEndpoint)) + } + + log.info("Creating S3 client with configuration: $builder") + return builder.build() + } +} diff --git a/src/test/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImplTest.groovy index a9cc2116d..e7d8a5b31 100644 --- a/src/test/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImplTest.groovy @@ -19,12 +19,21 @@ package io.seqera.wave.service.blob.impl import spock.lang.Specification +import java.util.concurrent.ExecutorService + import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.core.RegistryProxyService import io.seqera.wave.core.RoutePath import io.seqera.wave.model.ContainerCoordinates import io.seqera.wave.service.blob.BlobCacheInfo +import io.seqera.wave.service.blob.BlobStore import io.seqera.wave.test.AwsS3TestContainer +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest +import software.amazon.awssdk.services.s3.model.HeadObjectRequest +import software.amazon.awssdk.services.s3.model.HeadObjectResponse +import software.amazon.awssdk.services.s3.model.S3Exception + /** * * @author Paolo Di Tommaso @@ -82,4 +91,114 @@ class BlobCacheServiceImplTest extends Specification implements AwsS3TestContain ] } + def 'should return blob size when blob exists'() { + given: + def bucket = 's3://my-cache-bucket' + def expectedSize = 1024 + def s3Client = Mock(S3Client) + def blobCacheService = new BlobCacheServiceImpl(s3Client: s3Client, blobConfig: new BlobCacheConfig(storageBucket: bucket)) + and: + def route = Mock(RoutePath) { + getTargetPath() >> 'docker.io/repo/container/latest' + } + and: + final request = + HeadObjectRequest.builder() + .bucket('my-cache-bucket') + .key('docker.io/repo/container/latest') + .build() + + when: + def size = blobCacheService.getBlobSize(route) + + then: + 1 * s3Client.headObject(_) >> HeadObjectResponse.builder().contentLength(expectedSize).build() + and: + size == expectedSize + } + + def 'should return zero when blob does not exist'() { + given: + def bucket = 's3://my-cache-bucket' + def s3Client = Mock(S3Client) + def blobCacheService = new BlobCacheServiceImpl(s3Client: s3Client, blobConfig: new BlobCacheConfig(storageBucket: bucket)) + and: + def route = Mock(RoutePath) { + getTargetPath() >> 'docker.io/repo/container/latest' + } + and: + final request = + HeadObjectRequest.builder() + .bucket('my-cache-bucket') + .key('docker.io/repo/container/latest') + .build() + + when: + def size = blobCacheService.getBlobSize(route) + + then: + 1 * s3Client.headObject(request) >> { throw S3Exception.builder().message('Not Found').build() } + and: + size == -1L + } + + def 'should delete blob when blob exists'() { + given: + def bucket = 's3://my-cache-bucket/base/dir' + def s3Client = Mock(S3Client) + def blobCacheService = new BlobCacheServiceImpl(s3Client: s3Client, blobConfig: new BlobCacheConfig(storageBucket: bucket)) + and: + def route = Mock(RoutePath) { + getTargetPath() >> 'docker.io/repo/container/latest' + } + and: + def request = DeleteObjectRequest.builder() + .bucket('my-cache-bucket') + .key('base/dir/docker.io/repo/container/latest') + .build() + + when: + blobCacheService.deleteBlob(route) + then: + 1 * s3Client.deleteObject(request) >> { } + } + + def 'should return failed BlobCacheInfo when blob size mismatch'() { + given: + def executor = Mock(ExecutorService) + def s3Client = Mock(S3Client) + s3Client.headObject(_) >> HeadObjectResponse.builder().contentLength(1234L).build() + def blobStore = Mock(BlobStore) + def blobCacheService = new BlobCacheServiceImpl(s3Client: s3Client, blobConfig: new BlobCacheConfig(storageBucket: 's3://store/blobs/'), blobStore: blobStore, executor: executor, ) + def route = RoutePath.v2manifestPath(ContainerCoordinates.parse('ubuntu@sha256:aabbcc')) + def info = BlobCacheInfo.create('http://foo', [:], ['Content-Type':['foo'], 'Cache-Control': ['bar'], 'Content-Length': ['4321']]) + info = info.completed(0, 'Blob uploaded') + + when: + def result = blobCacheService.checkUploadedBlobSize(info, route) + + then: + !result.succeeded() + result.logs == "Mismatch cache size for object http://foo" + } + + def 'should return succeeded BlobCacheInfo when blob size matches'() { + given: + def executor = Mock(ExecutorService) + def s3Client = Mock(S3Client) + s3Client.headObject(_) >> HeadObjectResponse.builder().contentLength(4321L).build() + def blobStore = Mock(BlobStore) + def blobCacheService = new BlobCacheServiceImpl(s3Client: s3Client, blobConfig: new BlobCacheConfig(storageBucket: 's3://store/blobs/'), blobStore: blobStore, executor: executor) + def route = RoutePath.v2manifestPath(ContainerCoordinates.parse('ubuntu@sha256:aabbcc')) + def info = BlobCacheInfo.create('http://foo', [:], ['Content-Type':['foo'], 'Cache-Control': ['bar'], 'Content-Length': ['4321']]) + info = info.completed(0, 'Blob uploaded') + + when: + def result = blobCacheService.checkUploadedBlobSize(info, route) + + then: + result.succeeded() + result.logs == "Blob uploaded" + } + } diff --git a/src/test/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImplTest2.groovy b/src/test/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImplTest2.groovy index 5d8748140..0aeabf0fa 100644 --- a/src/test/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImplTest2.groovy +++ b/src/test/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImplTest2.groovy @@ -25,7 +25,6 @@ import io.micronaut.context.ApplicationContext import io.seqera.wave.core.RoutePath import io.seqera.wave.model.ContainerCoordinates import io.seqera.wave.test.AwsS3TestContainer - /** * * @author Paolo Di Tommaso @@ -43,7 +42,9 @@ class BlobCacheServiceImplTest2 extends Specification implements AwsS3TestContai 'wave.blobCache.storage.bucket': BUCKET, 'wave.blobCache.baseUrl': BASE_URL, 'wave.blobCache.storage.region': 'eu-west-1', - 'wave.blobCache.storage.endpoint': testEndpoint + 'wave.blobCache.storage.endpoint': testEndpoint, + 'wave.blobCache.storage.accessKey': 'accessKey', + 'wave.blobCache.storage.secretKey': 'secretKey' ] def ctx = ApplicationContext.run(PROPS) def service = ctx.getBean(BlobCacheServiceImpl) @@ -88,7 +89,9 @@ class BlobCacheServiceImplTest2 extends Specification implements AwsS3TestContai 'wave.blobCache.storage.bucket': BUCKET, 'wave.blobCache.baseUrl': BASE_URL, 'wave.blobCache.storage.region': 'eu-west-1', - 'wave.blobCache.storage.endpoint': testEndpoint + 'wave.blobCache.storage.endpoint': testEndpoint, + 'wave.blobCache.storage.accessKey': 'accessKey', + 'wave.blobCache.storage.secretKey': 'secretKey' ] def ctx = ApplicationContext.run(PROPS) def service = ctx.getBean(BlobCacheServiceImpl) diff --git a/src/test/groovy/io/seqera/wave/test/AwsS3TestContainer.groovy b/src/test/groovy/io/seqera/wave/test/AwsS3TestContainer.groovy index 77954fe4b..3ba68147f 100644 --- a/src/test/groovy/io/seqera/wave/test/AwsS3TestContainer.groovy +++ b/src/test/groovy/io/seqera/wave/test/AwsS3TestContainer.groovy @@ -40,8 +40,7 @@ trait AwsS3TestContainer { awsS3Container.start() log.debug "Started AWS S3 test container" } - - + String getAwsS3HostName(){ awsS3Container.getHost() } From 8282a4922fba642a1974acb472ed8c82d4ef3617 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 6 Jul 2024 17:25:44 +0200 Subject: [PATCH 5/5] Add http 429 error to auth service retry condition Signed-off-by: Paolo Di Tommaso --- .../io/seqera/wave/auth/RegistryAuthServiceImpl.groovy | 6 +++--- .../io/seqera/wave/auth/RegistryLookupServiceImpl.groovy | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy index 76df7e83c..ec6c4cb28 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy @@ -39,7 +39,7 @@ import io.seqera.wave.util.StringUtils import jakarta.inject.Inject import jakarta.inject.Singleton import static io.seqera.wave.WaveDefault.DOCKER_IO -import static io.seqera.wave.WaveDefault.HTTP_SERVER_ERRORS +import static io.seqera.wave.WaveDefault.HTTP_RETRYABLE_ERRORS /** * Implement Docker authentication & login service * @@ -116,7 +116,7 @@ class RegistryAuthServiceImpl implements RegistryAuthService { // retry strategy final retryable = Retryable .>of(httpConfig) - .retryIf( (response) -> response.statusCode() in HTTP_SERVER_ERRORS) + .retryIf( (response) -> response.statusCode() in HTTP_RETRYABLE_ERRORS) .onRetry((event) -> log.warn("Unable to connect '$endpoint' - event: $event}")) // make the request final response = retryable.apply(()-> httpClient.send(request, HttpResponse.BodyHandlers.ofString())) @@ -203,7 +203,7 @@ class RegistryAuthServiceImpl implements RegistryAuthService { // retry strategy final retryable = Retryable .>of(httpConfig) - .retryIf( (response) -> ((HttpResponse)response).statusCode() in HTTP_SERVER_ERRORS ) + .retryIf( (response) -> ((HttpResponse)response).statusCode() in HTTP_RETRYABLE_ERRORS ) .onRetry((event) -> log.warn("Unable to connect '$login' - event: $event")) // submit http request final response = retryable.apply(()-> httpClient.send(req, HttpResponse.BodyHandlers.ofString())) diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy index fde961428..db65227a1 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy @@ -34,7 +34,7 @@ import jakarta.inject.Inject import jakarta.inject.Singleton import static io.seqera.wave.WaveDefault.DOCKER_IO import static io.seqera.wave.WaveDefault.DOCKER_REGISTRY_1 -import static io.seqera.wave.WaveDefault.HTTP_SERVER_ERRORS +import static io.seqera.wave.WaveDefault.HTTP_RETRYABLE_ERRORS /** * Lookup service for container registry. The role of this component * is to registry the retrieve the registry authentication realm @@ -73,7 +73,7 @@ class RegistryLookupServiceImpl implements RegistryLookupService { // retry strategy final retryable = Retryable .>of(httpConfig) - .retryIf((response) -> response.statusCode() in HTTP_SERVER_ERRORS) + .retryIf((response) -> response.statusCode() in HTTP_RETRYABLE_ERRORS ) .onRetry((event) -> log.warn("Unable to connect '$endpoint' - event: $event")) // submit the request final response = retryable.apply(()-> httpClient.send(request, HttpResponse.BodyHandlers.ofString()))