From 9d31246bd2fe732beb0538358163d26a15fae7c0 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 9 Oct 2024 22:20:24 +0200 Subject: [PATCH 1/4] Replace mirrorRegistry with buildRepository Signed-off-by: Paolo Di Tommaso --- build.gradle | 2 +- .../controller/ContainerController.groovy | 66 ++++++++-------- .../persistence/PersistenceService.groovy | 8 ++ .../impl/LocalPersistenceService.groovy | 4 + .../impl/SurrealPersistenceService.groovy | 10 +++ .../scan/ContainerScanServiceImpl.groovy | 8 ++ .../validation/ValidationService.groovy | 4 +- .../validation/ValidationServiceImpl.groovy | 50 +++++++------ src/main/resources/application.yml | 1 + .../controller/ContainerControllerTest.groovy | 75 +++++++++---------- .../impl/SurrealPersistenceServiceTest.groovy | 23 ++++++ .../validation/ValidationServiceTest.groovy | 53 +++++++------ 12 files changed, 185 insertions(+), 119 deletions(-) diff --git a/build.gradle b/build.gradle index 530d84bf3..4c02f2a34 100644 --- a/build.gradle +++ b/build.gradle @@ -35,7 +35,7 @@ dependencies { compileOnly("io.micronaut:micronaut-http-validation") implementation("jakarta.persistence:jakarta.persistence-api:3.0.0") api 'io.seqera:lib-mail:1.0.0' - api 'io.seqera:wave-api:0.13.1' + api 'io.seqera:wave-api:0.13.2' api 'io.seqera:wave-utils:0.14.1' implementation("io.micronaut:micronaut-http-client") implementation("io.micronaut:micronaut-jackson-databind") diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index 6bd522a7c..dbbcdd454 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -73,6 +73,7 @@ import io.seqera.wave.service.request.ContainerStatusService import io.seqera.wave.service.request.TokenData import io.seqera.wave.service.scan.ContainerScanService import io.seqera.wave.service.validation.ValidationService +import io.seqera.wave.service.validation.ValidationServiceImpl import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.User import io.seqera.wave.tower.auth.JwtAuth @@ -240,14 +241,14 @@ class ContainerController { throw new BadRequestException("Attribute `nameStrategy` is not allowed by legacy container endpoint") // prevent the use of container file and freeze without a custom build repository - if( req.containerFile && req.freeze && !isCustomRepo0(req.buildRepository) && (!v2 || (v2 && !req.packages))) + if( req.containerFile && req.freeze && !validationService.isCustomRepo(req.buildRepository) && (!v2 || (v2 && !req.packages))) throw new BadRequestException("Attribute `buildRepository` must be specified when using freeze mode [1]") // prevent the use of container image and freeze without a custom build repository - if( req.containerImage && req.freeze && !isCustomRepo0(req.buildRepository) ) + if( req.containerImage && req.freeze && !validationService.isCustomRepo(req.buildRepository) ) throw new BadRequestException("Attribute `buildRepository` must be specified when using freeze mode [2]") - if( v2 && req.packages && req.freeze && !isCustomRepo0(req.buildRepository) && !buildConfig.defaultPublicRepository ) + if( v2 && req.packages && req.freeze && !validationService.isCustomRepo(req.buildRepository) && !buildConfig.defaultPublicRepository ) throw new BadRequestException("Attribute `buildRepository` must be specified when using freeze mode [3]") if( v2 && req.packages ) { @@ -280,18 +281,6 @@ class ContainerController { return HttpResponse.ok(resp) } - protected boolean isCustomRepo0(String repo) { - if( !repo ) - return false - if( buildConfig.defaultPublicRepository && repo.startsWith(buildConfig.defaultPublicRepository) ) - return false - if( buildConfig.defaultBuildRepository && repo.startsWith(buildConfig.defaultBuildRepository) ) - return false - if( buildConfig.defaultCacheRepository && repo.startsWith(buildConfig.defaultCacheRepository) ) - return false - return true - } - protected void storeContainerRequest0(SubmitContainerTokenRequest req, ContainerRequest data, TokenData token, String target, String ip) { try { final recrd = new WaveContainerRecord(req, data, target, ip, token.expiration) @@ -455,7 +444,7 @@ class ContainerController { scanId = build.scanId mirrorFlag = false } - else if( req.mirrorRegistry ) { + else if( req.mirror ) { final mirror = makeMirrorRequest(req, identity, digest) final track = checkMirror(mirror, identity, req.dryRun) targetImage = track.targetImage @@ -502,9 +491,16 @@ class ContainerController { protected MirrorRequest makeMirrorRequest(SubmitContainerTokenRequest request, PlatformId identity, String digest) { final coords = ContainerCoordinates.parse(request.containerImage) - if( coords.registry == request.mirrorRegistry ) - throw new BadRequestException("Source and target mirror registry as the same - offending value '${request.mirrorRegistry}'") - final targetImage = request.mirrorRegistry + '/' + coords.imageAndTag + final target = ContainerCoordinates.parse(request.buildRepository) + if( !coords.imageAndTag ) + throw new BadRequestException("Missing mirror source image - offending value '${request.containerImage}'") + if( !target.registry ) + throw new BadRequestException("Missing mirror target registry - offending value '${request.buildRepository}'") + if( coords.registry == target.registry ) + throw new BadRequestException("Source and target mirror registries are the same - offending value '${request.buildRepository}'") + final targetImage = target.repository + ? target.repository + '/' + coords.imageAndTag + : target.registry + '/' + coords.imageAndTag final configJson = inspectService.credentialsConfigJson(null, request.containerImage, targetImage, identity) final platform = request.containerPlatform ? ContainerPlatform.of(request.containerPlatform) @@ -582,41 +578,45 @@ class ContainerController { // check valid image name msg = validationService.checkContainerName(req.containerImage) if( msg ) throw new BadRequestException(msg) + // stop here for mirror request + if( req.mirror ) + return // check build repo - msg = validationService.checkBuildRepository(req.buildRepository, false) + if( !req.mirror ) + msg = validationService.checkBuildRepository(req.buildRepository, ValidationServiceImpl.RepoType.Build) if( msg ) throw new BadRequestException(msg) // check cache repository - msg = validationService.checkBuildRepository(req.cacheRepository, true) + msg = validationService.checkBuildRepository(req.cacheRepository, ValidationServiceImpl.RepoType.Cache) if( msg ) throw new BadRequestException(msg) } void validateMirrorRequest(SubmitContainerTokenRequest req, boolean v2) throws BadRequestException { - if( !req.mirrorRegistry ) + if( !req.mirror ) return // container mirror validation if( !v2 ) throw new BadRequestException("Container mirroring requires the use of v2 API") if( !req.containerImage ) - throw new BadRequestException("Attribute `containerImage` is required when specifying `mirrorRegistry`") + throw new BadRequestException("Attribute `containerImage` is required when specifying `mirror` mode") if( !req.towerAccessToken ) throw new BadRequestException("Container mirroring requires an authenticated request - specify the tower token attribute") if( req.freeze ) - throw new BadRequestException("Attribute `mirrorRegistry` and `freeze` conflict each other") + throw new BadRequestException("Attribute `mirror` and `freeze` conflict each other") if( req.containerFile ) - throw new BadRequestException("Attribute `mirrorRegistry` and `containerFile` conflict each other") + throw new BadRequestException("Attribute `mirror` and `containerFile` conflict each other") if( req.containerIncludes ) - throw new BadRequestException("Attribute `mirrorRegistry` and `containerIncludes` conflict each other") + throw new BadRequestException("Attribute `mirror` and `containerIncludes` conflict each other") if( req.containerConfig ) - throw new BadRequestException("Attribute `mirrorRegistry` and `containerConfig` conflict each other") + throw new BadRequestException("Attribute `mirror` and `containerConfig` conflict each other") + if( !req.buildRepository ) + throw new BadRequestException("Attribute `buildRepository` is required when specifying `mirror` mode") final coords = ContainerCoordinates.parse(req.containerImage) - if( coords.registry == req.mirrorRegistry ) - throw new BadRequestException("Source and target mirror registry as the same - offending value '${req.mirrorRegistry}'") - def msg = validationService.checkMirrorRegistry(req.mirrorRegistry) + final target = ContainerCoordinates.parse(req.buildRepository) + if( coords.registry == target.registry ) + throw new BadRequestException("Source and target mirror registry are the same - offending value '${req.buildRepository}'") + def msg = validationService.checkBuildRepository(req.buildRepository, ValidationServiceImpl.RepoType.Mirror) if( msg ) throw new BadRequestException(msg) - if( !isCustomRepo0(req.mirrorRegistry) ) { - throw new BadRequestException("Not allowed mirror registry - offending value '${req.mirrorRegistry}'") - } } @Error(exception = AuthorizationException.class) diff --git a/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy index c36a4c19f..1f09cce27 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy @@ -105,6 +105,14 @@ interface PersistenceService { */ WaveScanRecord loadScanRecord(String scanId) + /** + * Check if a scan record exist + * + * @param scanId The Id of the scan to check + * @return {@code true} if the scan record with the specified id exists or {@code false} otherwise + */ + boolean existsScanRecord(String scanId) + /** * Load a mirror state record * diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy index d1c038d18..445bc7a5b 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy @@ -83,6 +83,10 @@ class LocalPersistenceService implements PersistenceService { requestStore.get(token) } + @Override + boolean existsScanRecord(String scanId) { + scanStore.containsKey(scanId) + } @Override void saveScanRecord(WaveScanRecord scanRecord) { diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy index b8acf2070..f16e6e961 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy @@ -258,6 +258,16 @@ class SurrealPersistenceService implements PersistenceService { json.replaceFirst(/"vulnerabilities":\s*\[]/, value) } + @Override + boolean existsScanRecord(String scanId) { + final statement = "SELECT count() FROM wave_scan where id == 'wave_scan:⟨$scanId⟩'" + final json = surrealDb.sqlAsString(getAuthorization(), statement) + final type = new TypeReference>>() {} + final data= json ? JacksonHelper.fromJson(json, type) : null + final result = data && data[0].result ? data[0].result[0].count==1 : false + return result + } + @Override WaveScanRecord loadScanRecord(String scanId) { if( !scanId ) diff --git a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy index b71e340ec..000abaece 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy @@ -128,6 +128,10 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler VALID_PROTOCOLS = ['http','https'] + @Inject + private BuildConfig buildConfig + @Override String checkEndpoint(String endpoint) { final scheme = StringUtils.getUrlProtocol(endpoint) @@ -74,38 +81,39 @@ class ValidationServiceImpl implements ValidationService { } @Override - String checkBuildRepository(String repo, boolean cache) { - if( !repo ) + String checkBuildRepository(String repo, RepoType type ) { + if( !repo && type!=RepoType.Mirror ) return null - final type = cache ? "build cache" : "build" + // repo is required when using mirror more + if( !repo && type==RepoType.Mirror ) + return "Missing target build repository required by 'mirror' mode" + final typeStr = type==RepoType.Cache ? "build cache" : "build" // check does not start with a protocol prefix final prot = StringUtils.getUrlProtocol(repo) if( prot ) - return "Container ${type} repository should not include any protocol prefix - offending value: $repo" + return "Container ${typeStr} repository should not include any protocol prefix - offending value: '$repo'" // check no tag is included final coords = ContainerCoordinates.parse(repo) - if( !coords.repository ) - return "Container ${type} repository is invalid or incomplete - offending value: $repo" + if( !coords.repository && type!=RepoType.Mirror ) + return "Container ${typeStr} repository is invalid or incomplete - offending value: '$repo'" if( coords.reference && repo.endsWith(":${coords.reference}") ) - return "Container ${type} repository should not include any tag suffix - offending value: $repo" + return "Container ${typeStr} repository should not include any tag suffix - offending value: '$repo'" + if( type==RepoType.Mirror && !isCustomRepo(coords.registry) ) + return "Mirror registry not allowed - offending value '${repo}'" else return null } - @Override - String checkMirrorRegistry(String registry) { - if( !registry ) - return null - final prot = StringUtils.getUrlProtocol(registry) - if( prot ) - return "Mirror registry should not include any protocol prefix - offending value: $registry" - // check no tag is included - final coords = ContainerCoordinates.parse(registry) - if( coords.repository ) - return "Mirror registry syntax is invalid - offending value: ${registry}" - if( coords.registry == 'wave.seqera.io' || coords.registry?.contains('.wave.seqera.io') ) - return "Mirror registry not allowed - offending value: ${registry}" - return null + boolean isCustomRepo(String repo) { + if( !repo ) + return false + if( buildConfig.defaultPublicRepository && repo.startsWith(buildConfig.defaultPublicRepository) ) + return false + if( buildConfig.defaultBuildRepository && repo.startsWith(buildConfig.defaultBuildRepository) ) + return false + if( buildConfig.defaultCacheRepository && repo.startsWith(buildConfig.defaultCacheRepository) ) + return false + return true } } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index e5a3b0832..38d685608 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -71,6 +71,7 @@ wave: singularity-image-arm64: "quay.io/singularity/singularity:v3.11.4-slim-arm64" repo: "195996028523.dkr.ecr.eu-west-1.amazonaws.com/wave/build/dev" cache: "195996028523.dkr.ecr.eu-west-1.amazonaws.com/wave/build/cache" + public-repo: "community.wave.seqera.io" # note changing this should be matching the micronaut.server.idle-timeout (see above) timeout: 900s status: diff --git a/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy index 94ec47fa5..76101b5cf 100644 --- a/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy @@ -61,6 +61,7 @@ import io.seqera.wave.service.persistence.PersistenceService import io.seqera.wave.service.persistence.WaveContainerRecord import io.seqera.wave.service.request.ContainerRequestService import io.seqera.wave.service.request.TokenData +import io.seqera.wave.service.validation.ValidationService import io.seqera.wave.service.validation.ValidationServiceImpl import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.User @@ -92,6 +93,9 @@ class ContainerControllerTest extends Specification { @Inject RegistryProxyService proxyRegistry + @Inject + ValidationService validationService + @MockBean(JobServiceImpl) JobService mockJobService() { Mock(JobService) @@ -266,24 +270,31 @@ class ContainerControllerTest extends Specification { def controller = new ContainerController(mirrorService: mirrorService, inspectService: inspectService, registryProxyService: proxyRegistry, buildConfig: buildConfig, inclusionService: Mock(ContainerInclusionService)) def user = new PlatformId(new User(id: 100)) def req = new SubmitContainerTokenRequest( - containerImage: 'docker.io/source/image:latest', + containerImage: SOURCE, containerPlatform: 'arm64', - mirrorRegistry: 'quay.io' + buildRepository: BUILD, + mirror: true ) when: def data = controller.makeRequestData(req, user, "") then: - 1 * proxyRegistry.getImageDigest('docker.io/source/image:latest', user) >> 'sha256:12345' - 1 * proxyRegistry.getImageDigest('quay.io/source/image:latest', user) >> null + 1 * proxyRegistry.getImageDigest(SOURCE, user) >> 'sha256:12345' + 1 * proxyRegistry.getImageDigest(TARGET, user) >> null and: data.identity.userId == 100 - data.containerImage == 'quay.io/source/image:latest' + data.containerImage == TARGET data.platform.toString() == 'linux/arm64' data.buildId =~ /mr-.+/ data.buildNew !data.freeze data.mirror + + where: + SOURCE | BUILD | TARGET + 'docker.io/source/image:latest' | 'quay.io' | 'quay.io/source/image:latest' + 'docker.io/source/image:latest' | 'quay.io/lib' | 'quay.io/lib/source/image:latest' + 'docker.io/source/image:latest' | 'quay.io/lib/' | 'quay.io/lib/source/image:latest' } def 'should create build request' () { @@ -452,82 +463,70 @@ class ContainerControllerTest extends Specification { def 'should validate mirror request' () { given: - def validation = new ValidationServiceImpl() def pairing = Mock(PairingService) def channel = Mock(PairingChannel) - def controller = new ContainerController(validationService: validation, pairingService: pairing, pairingChannel: channel) + def controller = new ContainerController(validationService: validationService, pairingService: pairing, pairingChannel: channel) def err when: - controller.validateMirrorRequest(new SubmitContainerTokenRequest(containerImage: 'foo:latest'), false) - then: - noExceptionThrown() - - when: - controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirrorRegistry: 'quay.io'), false) + controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirror: true, buildRepository: 'quay.io'), false) then: err = thrown(BadRequestException) err.message == 'Container mirroring requires the use of v2 API' when: - controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirrorRegistry: 'quay.io'), true) + controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirror: true, buildRepository: 'quay.io'), true) then: err = thrown(BadRequestException) - err.message == 'Attribute `containerImage` is required when specifying `mirrorRegistry`' + err.message == 'Attribute `containerImage` is required when specifying `mirror` mode' when: - controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirrorRegistry: 'quay.io', containerImage: 'docker.io/foo'), true) + controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirror: true, buildRepository: 'quay.io', containerImage: 'docker.io/foo'), true) then: err = thrown(BadRequestException) err.message == 'Container mirroring requires an authenticated request - specify the tower token attribute' when: - controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirrorRegistry: 'docker.io', containerImage: 'docker.io/foo', towerAccessToken: 'xyz'), true) - then: - err = thrown(BadRequestException) - err.message == "Source and target mirror registry as the same - offending value 'docker.io'" - - when: - controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirrorRegistry: 'docker.io', containerImage: 'foo', towerAccessToken: 'xyz'), true) + controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirror: true, buildRepository: 'docker.io', containerImage: 'docker.io/foo', towerAccessToken: 'xyz'), true) then: err = thrown(BadRequestException) - err.message == "Source and target mirror registry as the same - offending value 'docker.io'" + err.message == "Source and target mirror registry are the same - offending value 'docker.io'" when: - controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirrorRegistry: 'quay.io', containerImage: 'docker.io/foo', towerAccessToken: 'xyz', containerFile: 'content'), true) + controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirror: true, buildRepository: 'docker.io', containerImage: 'foo', towerAccessToken: 'xyz'), true) then: err = thrown(BadRequestException) - err.message == "Attribute `mirrorRegistry` and `containerFile` conflict each other" + err.message == "Source and target mirror registry are the same - offending value 'docker.io'" when: - controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirrorRegistry: 'quay.io', containerImage: 'docker.io/foo', towerAccessToken: 'xyz', freeze: true), true) + controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirror: true, buildRepository: 'quay.io', containerImage: 'docker.io/foo', towerAccessToken: 'xyz', containerFile: 'content'), true) then: err = thrown(BadRequestException) - err.message == "Attribute `mirrorRegistry` and `freeze` conflict each other" + err.message == "Attribute `mirror` and `containerFile` conflict each other" when: - controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirrorRegistry: 'quay.io', containerImage: 'docker.io/foo', towerAccessToken: 'xyz', containerIncludes: ['include']), true) + controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirror: true, buildRepository: 'quay.io', containerImage: 'docker.io/foo', towerAccessToken: 'xyz', freeze: true), true) then: err = thrown(BadRequestException) - err.message == "Attribute `mirrorRegistry` and `containerIncludes` conflict each other" + err.message == "Attribute `mirror` and `freeze` conflict each other" when: - controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirrorRegistry: 'quay.io', containerImage: 'docker.io/foo', towerAccessToken: 'xyz', containerConfig: new ContainerConfig(entrypoint: ['foo'])), true) + controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirror: true, buildRepository: 'quay.io', containerImage: 'docker.io/foo', towerAccessToken: 'xyz', containerIncludes: ['include']), true) then: err = thrown(BadRequestException) - err.message == "Attribute `mirrorRegistry` and `containerConfig` conflict each other" + err.message == "Attribute `mirror` and `containerIncludes` conflict each other" when: - controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirrorRegistry: 'quay.io/bar', containerImage: 'docker.io/foo', towerAccessToken: 'xyz'), true) + controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirror: true, buildRepository: 'quay.io', containerImage: 'docker.io/foo', towerAccessToken: 'xyz', containerConfig: new ContainerConfig(entrypoint: ['foo'])), true) then: err = thrown(BadRequestException) - err.message == "Mirror registry syntax is invalid - offending value: quay.io/bar" + err.message == "Attribute `mirror` and `containerConfig` conflict each other" when: - controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirrorRegistry: 'community.wave.seqera.io', containerImage: 'docker.io/foo', towerAccessToken: 'xyz'), true) + controller.validateMirrorRequest(new SubmitContainerTokenRequest(mirror: true, buildRepository: 'community.wave.seqera.io', containerImage: 'docker.io/foo', towerAccessToken: 'xyz'), true) then: err = thrown(BadRequestException) - err.message == "Mirror registry not allowed - offending value: community.wave.seqera.io" + err.message == "Mirror registry not allowed - offending value 'community.wave.seqera.io'" } def 'should create response with conda packages' () { @@ -541,7 +540,7 @@ class ContainerControllerTest extends Specification { def persistence = Mock(PersistenceService) def controller = new ContainerController(freezeService: freeze, buildService: builder, inspectService: dockerAuth, registryProxyService: proxyRegistry, buildConfig: buildConfig, inclusionService: Mock(ContainerInclusionService), - addressResolver: addressResolver, containerService: tokenService, persistenceService: persistence, serverUrl: 'http://wave.com') + addressResolver: addressResolver, containerService: tokenService, persistenceService: persistence, validationService: validationService, serverUrl: 'http://wave.com') when:'packages with conda' def CHANNELS = ['conda-forge', 'defaults'] @@ -569,7 +568,7 @@ class ContainerControllerTest extends Specification { def 'should throw BadRequestException when more than one artifact (container image, container file or packages) is provided in the request' () { given: - def controller = new ContainerController(inclusionService: Mock(ContainerInclusionService), allowAnonymous: false) + def controller = new ContainerController(validationService: validationService, inclusionService: Mock(ContainerInclusionService), allowAnonymous: false) when: 'container access token is not provided' def req = new SubmitContainerTokenRequest(packages: new PackagesSpec()) diff --git a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy index 61a82e1d7..553a8412d 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy @@ -303,6 +303,8 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe .sqlAsMap(auth, "select * from wave_scan_vuln") .result .size() == 3 + and: + persistence.existsScanRecord(SCAN_ID) when: def SCAN_ID2 = 'b2' @@ -322,6 +324,27 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe .size() == 4 } + def 'should save a scan and check it exists' () { + given: + def persistence = applicationContext.getBean(SurrealPersistenceService) + def auth = persistence.getAuthorization() + def surrealDb = applicationContext.getBean(SurrealClient) + def NOW = Instant.now() + def SCAN_ID = 'a1' + def BUILD_ID = '100' + def CONTAINER_IMAGE = 'docker.io/my/repo:container1234' + def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') + def scan = new WaveScanRecord(SCAN_ID, BUILD_ID, null, null, CONTAINER_IMAGE, NOW, Duration.ofSeconds(10), 'SUCCEEDED', [CVE1], null, null) + + expect: + !persistence.existsScanRecord(SCAN_ID) + + when: + persistence.saveScanRecord(scan) + then: + persistence.existsScanRecord(SCAN_ID) + } + //== mirror records tests void "should save and load a mirror record by id"() { diff --git a/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceTest.groovy index e2e4d7c4d..12ba6c1e1 100644 --- a/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceTest.groovy @@ -21,9 +21,13 @@ package io.seqera.wave.service.validation import spock.lang.Specification import spock.lang.Unroll +import io.micronaut.test.annotation.MockBean import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.configuration.BuildConfig import jakarta.inject.Inject +import static io.seqera.wave.service.validation.ValidationServiceImpl.RepoType.* + /** * * @author Paolo Di Tommaso @@ -31,6 +35,13 @@ import jakarta.inject.Inject @MicronautTest class ValidationServiceTest extends Specification { + @MockBean(BuildConfig) + BuildConfig buildConfig() { + Mock(BuildConfig) { + defaultPublicRepository >> 'public.repo.io' + } + } + @Inject ValidationService validationService @@ -81,33 +92,27 @@ class ValidationServiceTest extends Specification { where: CONTAINER | TYPE | EXPECTED - null | false | null - 'foo.com/ubuntu' | false | null - 'foo.com' | false | 'Container build repository is invalid or incomplete - offending value: foo.com' - 'http://foo.com' | false | 'Container build repository should not include any protocol prefix - offending value: http://foo.com' - 'foo.com/ubuntu:latest' | false | 'Container build repository should not include any tag suffix - offending value: foo.com/ubuntu:latest' - and: - null | true | null - 'foo.com/ubuntu' | true | null - 'foo.com' | true | 'Container build cache repository is invalid or incomplete - offending value: foo.com' - 'http://foo.com' | true | 'Container build cache repository should not include any protocol prefix - offending value: http://foo.com' - 'foo.com/ubuntu:latest' | true | 'Container build cache repository should not include any tag suffix - offending value: foo.com/ubuntu:latest' + null | Build | null + 'foo.com/ubuntu' | Build | null + 'foo.com' | Build | "Container build repository is invalid or incomplete - offending value: 'foo.com'" + 'http://foo.com' | Build | "Container build repository should not include any protocol prefix - offending value: 'http://foo.com'" + 'foo.com/ubuntu:latest' | Build | "Container build repository should not include any tag suffix - offending value: 'foo.com/ubuntu:latest'" - } + and: + null | Cache | null + 'foo.com/ubuntu' | Cache | null + 'foo.com' | Cache | "Container build cache repository is invalid or incomplete - offending value: 'foo.com'" + 'http://foo.com' | Cache | "Container build cache repository should not include any protocol prefix - offending value: 'http://foo.com'" + 'foo.com/ubuntu:latest' | Cache | "Container build cache repository should not include any tag suffix - offending value: 'foo.com/ubuntu:latest'" - @Unroll - def 'should check registry' () { - expect: - validationService.checkMirrorRegistry(REG)==EXPECTED + and: + null | Mirror | "Missing target build repository required by 'mirror' mode" + 'foo.com' | Mirror | null + 'foo.com/ubuntu' | Mirror | null + 'http://foo.com' | Mirror | "Container build repository should not include any protocol prefix - offending value: 'http://foo.com'" + 'foo.com/ubuntu:latest' | Mirror | "Container build repository should not include any tag suffix - offending value: 'foo.com/ubuntu:latest'" + 'public.repo.io' | Mirror | "Mirror registry not allowed - offending value 'public.repo.io'" - where: - REG | EXPECTED - null | null - 'docker.io' | null - 'docker.io/foo' | 'Mirror registry syntax is invalid - offending value: docker.io/foo' - 'docker://foo.io' | 'Mirror registry should not include any protocol prefix - offending value: docker://foo.io' - 'wave.seqera.io' | 'Mirror registry not allowed - offending value: wave.seqera.io' - 'cr.wave.seqera.io' | 'Mirror registry not allowed - offending value: cr.wave.seqera.io' } } From 1a27aee91bec974e1b31e71be065369d8e33b7ed Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 9 Oct 2024 23:01:20 +0200 Subject: [PATCH 2/4] Update container request typespec model Signed-off-by: Paolo Di Tommaso --- typespec/models/ContainerRequest.tsp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typespec/models/ContainerRequest.tsp b/typespec/models/ContainerRequest.tsp index 6edd6b25d..38c4ef56a 100644 --- a/typespec/models/ContainerRequest.tsp +++ b/typespec/models/ContainerRequest.tsp @@ -18,7 +18,7 @@ model ContainerRequest { format: "sif" | "docker"; freeze?: boolean; nameStrategy?: "none" | "tagPrefix" | "imageSuffix"; - mirrorRegistry?: string; + mirror?: boolean; packages?: Packages; scanMode?: ScanMode; scanLevels?: ScanLevel[]; From 6270b6c5540948b0fceac5de54b306c07e4bcc67 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 9 Oct 2024 23:02:08 +0200 Subject: [PATCH 3/4] Update scan state duration to 5 days Signed-off-by: Paolo Di Tommaso --- src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy index 3d1e9cdf0..b9f4f48fd 100644 --- a/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy @@ -72,7 +72,7 @@ class ScanConfig { @Value('${wave.scan.retry-attempts:1}') int retryAttempts - @Value('${wave.scan.status.duration:1h}') + @Value('${wave.scan.status.duration:5d}') Duration statusDuration @Value('${wave.scan.id.duration:7d}') From 2b20eeedcfe17eb0a38d1e3e6bdc6bbf4f976bfd Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Wed, 9 Oct 2024 23:03:14 +0200 Subject: [PATCH 4/4] Move MirrorConfig to config package (#668) Signed-off-by: munishchouhan --- .../wave/{service/mirror => configuration}/MirrorConfig.groovy | 2 +- src/main/groovy/io/seqera/wave/service/job/JobFactory.groovy | 2 +- src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy | 2 +- .../groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy | 2 +- .../io/seqera/wave/service/mirror/MirrorStateStore.groovy | 2 ++ .../wave/service/mirror/strategy/DockerMirrorStrategy.groovy | 2 +- .../wave/service/mirror/strategy/KubeMirrorStrategy.groovy | 2 +- .../groovy/io/seqera/wave/service/job/JobFactoryTest.groovy | 2 +- .../groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy | 2 +- .../service/mirror/strategy/DockerMirrorStrategyTest.groovy | 2 +- 10 files changed, 11 insertions(+), 9 deletions(-) rename src/main/groovy/io/seqera/wave/{service/mirror => configuration}/MirrorConfig.groovy (97%) diff --git a/src/main/groovy/io/seqera/wave/service/mirror/MirrorConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/MirrorConfig.groovy similarity index 97% rename from src/main/groovy/io/seqera/wave/service/mirror/MirrorConfig.groovy rename to src/main/groovy/io/seqera/wave/configuration/MirrorConfig.groovy index 52be93819..a6eba4afa 100644 --- a/src/main/groovy/io/seqera/wave/service/mirror/MirrorConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/MirrorConfig.groovy @@ -16,7 +16,7 @@ * along with this program. If not, see . */ -package io.seqera.wave.service.mirror +package io.seqera.wave.configuration import java.time.Duration diff --git a/src/main/groovy/io/seqera/wave/service/job/JobFactory.groovy b/src/main/groovy/io/seqera/wave/service/job/JobFactory.groovy index 3be4599f1..17a519f11 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobFactory.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobFactory.groovy @@ -26,7 +26,7 @@ import groovy.transform.CompileStatic import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.service.builder.BuildRequest -import io.seqera.wave.service.mirror.MirrorConfig +import io.seqera.wave.configuration.MirrorConfig import io.seqera.wave.service.mirror.MirrorRequest import io.seqera.wave.service.scan.ScanRequest import jakarta.inject.Inject diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy index 6fa0a18b2..32fc40617 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy @@ -26,7 +26,7 @@ import io.kubernetes.client.openapi.models.V1Pod import io.kubernetes.client.openapi.models.V1PodList import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.ScanConfig -import io.seqera.wave.service.mirror.MirrorConfig +import io.seqera.wave.configuration.MirrorConfig /** * Defines Kubernetes operations diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index 1a91e9a39..4d4c23106 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -48,7 +48,7 @@ import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.BuildConfig import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.core.ContainerPlatform -import io.seqera.wave.service.mirror.MirrorConfig +import io.seqera.wave.configuration.MirrorConfig import io.seqera.wave.service.scan.Trivy import jakarta.inject.Inject import jakarta.inject.Singleton diff --git a/src/main/groovy/io/seqera/wave/service/mirror/MirrorStateStore.groovy b/src/main/groovy/io/seqera/wave/service/mirror/MirrorStateStore.groovy index 15cce21c5..31e61ba06 100644 --- a/src/main/groovy/io/seqera/wave/service/mirror/MirrorStateStore.groovy +++ b/src/main/groovy/io/seqera/wave/service/mirror/MirrorStateStore.groovy @@ -18,6 +18,8 @@ package io.seqera.wave.service.mirror +import io.seqera.wave.configuration.MirrorConfig + import java.time.Duration import groovy.transform.CompileStatic diff --git a/src/main/groovy/io/seqera/wave/service/mirror/strategy/DockerMirrorStrategy.groovy b/src/main/groovy/io/seqera/wave/service/mirror/strategy/DockerMirrorStrategy.groovy index 9fa346d66..ff96cea4c 100644 --- a/src/main/groovy/io/seqera/wave/service/mirror/strategy/DockerMirrorStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/mirror/strategy/DockerMirrorStrategy.groovy @@ -26,7 +26,7 @@ import groovy.json.JsonOutput import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value -import io.seqera.wave.service.mirror.MirrorConfig +import io.seqera.wave.configuration.MirrorConfig import io.seqera.wave.service.mirror.MirrorRequest import jakarta.inject.Inject import jakarta.inject.Singleton diff --git a/src/main/groovy/io/seqera/wave/service/mirror/strategy/KubeMirrorStrategy.groovy b/src/main/groovy/io/seqera/wave/service/mirror/strategy/KubeMirrorStrategy.groovy index 40fa62ed2..7ff9c08c5 100644 --- a/src/main/groovy/io/seqera/wave/service/mirror/strategy/KubeMirrorStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/mirror/strategy/KubeMirrorStrategy.groovy @@ -29,7 +29,7 @@ import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Requires import io.seqera.wave.exception.BadRequestException import io.seqera.wave.service.k8s.K8sService -import io.seqera.wave.service.mirror.MirrorConfig +import io.seqera.wave.configuration.MirrorConfig import io.seqera.wave.service.mirror.MirrorRequest import jakarta.inject.Inject import jakarta.inject.Singleton diff --git a/src/test/groovy/io/seqera/wave/service/job/JobFactoryTest.groovy b/src/test/groovy/io/seqera/wave/service/job/JobFactoryTest.groovy index 947eb5f90..458fb1443 100644 --- a/src/test/groovy/io/seqera/wave/service/job/JobFactoryTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/job/JobFactoryTest.groovy @@ -28,7 +28,7 @@ import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.builder.BuildRequest -import io.seqera.wave.service.mirror.MirrorConfig +import io.seqera.wave.configuration.MirrorConfig import io.seqera.wave.service.mirror.MirrorRequest import io.seqera.wave.service.scan.ScanRequest import io.seqera.wave.tower.PlatformId diff --git a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy index 5ce208064..07dd06f02 100644 --- a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy @@ -42,7 +42,7 @@ import io.micronaut.context.annotation.Replaces import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.ScanConfig -import io.seqera.wave.service.mirror.MirrorConfig +import io.seqera.wave.configuration.MirrorConfig /** * * @author Paolo Di Tommaso diff --git a/src/test/groovy/io/seqera/wave/service/mirror/strategy/DockerMirrorStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/mirror/strategy/DockerMirrorStrategyTest.groovy index 1e0bf8d35..f34064211 100644 --- a/src/test/groovy/io/seqera/wave/service/mirror/strategy/DockerMirrorStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/mirror/strategy/DockerMirrorStrategyTest.groovy @@ -22,7 +22,7 @@ import spock.lang.Specification import java.nio.file.Path -import io.seqera.wave.service.mirror.MirrorConfig +import io.seqera.wave.configuration.MirrorConfig /** *